From 3272924dddacd048069e87f7d2f1f64b019a6320 Mon Sep 17 00:00:00 2001 From: Urmish Date: Mon, 25 Apr 2022 14:22:18 -0500 Subject: [PATCH 01/16] First 2 prompts for blimp --- .../blimp/adjunct_island/templates.yaml | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 promptsource/templates/blimp/adjunct_island/templates.yaml diff --git a/promptsource/templates/blimp/adjunct_island/templates.yaml b/promptsource/templates/blimp/adjunct_island/templates.yaml new file mode 100644 index 000000000..62939c33c --- /dev/null +++ b/promptsource/templates/blimp/adjunct_island/templates.yaml @@ -0,0 +1,40 @@ +dataset: blimp +subset: adjunct_island +templates: + 0e5b5e79-cdea-47d3-b31c-c061162d3203: !Template + answer_choices: Sentence 1||| Sentence 2 + id: 0e5b5e79-cdea-47d3-b31c-c061162d3203 + jinja: 'Between the first sentence and the second sentence, which one is grammatically + correct? + + Sentence 1 - {{sentence_good}} + + Sentence 2 - {{sentence_bad}} + + + Answer - {{answer_choices[0] }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: adjunct_good_first_1 + reference: '' + 7e8b5aa4-be3d-48f1-ba58-ab9eea3c7fc0: !Template + answer_choices: Sentence 1||| Sentence 2 + id: 7e8b5aa4-be3d-48f1-ba58-ab9eea3c7fc0 + jinja: 'Between the first sentence and the second sentence, which one is grammatically + correct? + + Sentence 1 - {{sentence_bad}} + + Sentence 2 - {{sentence_good}} + + + Answer - {{answer_choices[1] }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: [] + original_task: false + name: adjunct_bad_first_1 + reference: '' From 74450d760b49ef8e809720d1dc5fea1975ffc38f Mon Sep 17 00:00:00 2001 From: Urmish Date: Mon, 25 Apr 2022 14:32:54 -0500 Subject: [PATCH 02/16] Fixed some template issues that led to conflicting details --- .../templates/blimp/adjunct_island/templates.yaml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/promptsource/templates/blimp/adjunct_island/templates.yaml b/promptsource/templates/blimp/adjunct_island/templates.yaml index 62939c33c..3a413b55b 100644 --- a/promptsource/templates/blimp/adjunct_island/templates.yaml +++ b/promptsource/templates/blimp/adjunct_island/templates.yaml @@ -2,7 +2,7 @@ dataset: blimp subset: adjunct_island templates: 0e5b5e79-cdea-47d3-b31c-c061162d3203: !Template - answer_choices: Sentence 1||| Sentence 2 + answer_choices: null id: 0e5b5e79-cdea-47d3-b31c-c061162d3203 jinja: 'Between the first sentence and the second sentence, which one is grammatically correct? @@ -12,7 +12,7 @@ templates: Sentence 2 - {{sentence_bad}} - Answer - {{answer_choices[0] }}' + Answer - Sentence 1' metadata: !TemplateMetadata choices_in_prompt: true metrics: @@ -21,7 +21,7 @@ templates: name: adjunct_good_first_1 reference: '' 7e8b5aa4-be3d-48f1-ba58-ab9eea3c7fc0: !Template - answer_choices: Sentence 1||| Sentence 2 + answer_choices: null id: 7e8b5aa4-be3d-48f1-ba58-ab9eea3c7fc0 jinja: 'Between the first sentence and the second sentence, which one is grammatically correct? @@ -31,10 +31,11 @@ templates: Sentence 2 - {{sentence_good}} - Answer - {{answer_choices[1] }}' + Answer - Sentence 2' metadata: !TemplateMetadata choices_in_prompt: true - metrics: [] - original_task: false + metrics: + - Accuracy + original_task: true name: adjunct_bad_first_1 reference: '' From b6f01dbde1fc6c1e0760819bfa193ad4fd3db8fe Mon Sep 17 00:00:00 2001 From: Urmish Date: Mon, 25 Apr 2022 14:38:13 -0500 Subject: [PATCH 03/16] Updated templates to avoid packing errors --- .../templates/blimp/adjunct_island/templates.yaml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/promptsource/templates/blimp/adjunct_island/templates.yaml b/promptsource/templates/blimp/adjunct_island/templates.yaml index 3a413b55b..95bbac6a7 100644 --- a/promptsource/templates/blimp/adjunct_island/templates.yaml +++ b/promptsource/templates/blimp/adjunct_island/templates.yaml @@ -12,7 +12,9 @@ templates: Sentence 2 - {{sentence_bad}} - Answer - Sentence 1' + Answer - ||| + + Sentence 1' metadata: !TemplateMetadata choices_in_prompt: true metrics: @@ -31,7 +33,9 @@ templates: Sentence 2 - {{sentence_good}} - Answer - Sentence 2' + Answer - ||| + + Sentence 2' metadata: !TemplateMetadata choices_in_prompt: true metrics: From 0a1689cc3e4eff9f9d13ec501a7059115a8f4506 Mon Sep 17 00:00:00 2001 From: Urmish Date: Mon, 25 Apr 2022 14:55:10 -0500 Subject: [PATCH 04/16] Added 2 new prompts with a different style of asking question --- .../blimp/adjunct_island/templates.yaml | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/promptsource/templates/blimp/adjunct_island/templates.yaml b/promptsource/templates/blimp/adjunct_island/templates.yaml index 95bbac6a7..f407cf80e 100644 --- a/promptsource/templates/blimp/adjunct_island/templates.yaml +++ b/promptsource/templates/blimp/adjunct_island/templates.yaml @@ -22,6 +22,28 @@ templates: original_task: true name: adjunct_good_first_1 reference: '' + 2599b172-d01c-4945-8ad2-5100c5eece1a: !Template + answer_choices: null + id: 2599b172-d01c-4945-8ad2-5100c5eece1a + jinja: 'Amongst the two sentences presented below, which one is syntactically + correct? + + + Sentence 1 - {{sentence_bad}} + + Sentence 2 - {{sentence_good}} + + + Answer - ||| + + Sentence 2' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: adjunct_bad_first_2 + reference: '' 7e8b5aa4-be3d-48f1-ba58-ab9eea3c7fc0: !Template answer_choices: null id: 7e8b5aa4-be3d-48f1-ba58-ab9eea3c7fc0 @@ -43,3 +65,25 @@ templates: original_task: true name: adjunct_bad_first_1 reference: '' + d6195bf0-b9df-456f-b66c-f22fc150aedd: !Template + answer_choices: null + id: d6195bf0-b9df-456f-b66c-f22fc150aedd + jinja: 'Amongst the two sentences presented below, which one is syntactically + correct? + + + Sentence 1 - {{sentence_good}} + + Sentence 2 - {{sentence_bad}} + + + Answer - ||| + + Sentence 1' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: adjunct_good_first_2 + reference: '' From f651092b931336d9102ee13a196ca4515a06931f Mon Sep 17 00:00:00 2001 From: cookie Date: Mon, 25 Apr 2022 15:58:57 -0400 Subject: [PATCH 05/16] Add Najoung\'s templates --- .../blimp/adjunct_island/templates.yaml | 76 +++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/promptsource/templates/blimp/adjunct_island/templates.yaml b/promptsource/templates/blimp/adjunct_island/templates.yaml index f407cf80e..4013f34be 100644 --- a/promptsource/templates/blimp/adjunct_island/templates.yaml +++ b/promptsource/templates/blimp/adjunct_island/templates.yaml @@ -87,3 +87,79 @@ templates: original_task: true name: adjunct_good_first_2 reference: '' + 3d2821a7-1c9c-44dd-ad23-e964b0998d1b: !Template + answer_choices: A ||| B + id: 3d2821a7-1c9c-44dd-ad23-e964b0998d1b + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'A/B choice 2 (bad-good, answer: B)' + reference: '' + 41594c98-baea-4b1f-a631-27842cd363fd: !Template + answer_choices: B ||| A + id: 41594c98-baea-4b1f-a631-27842cd363fd + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'B/A choice 1 (good-bad, answer: A)' + reference: '' + 88abc121-43ea-4b67-92e3-d762884d391d: !Template + answer_choices: B ||| A + id: 88abc121-43ea-4b67-92e3-d762884d391d + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'B/A choice, 2 (bad-good, answer: B)' + reference: '' + 8f354d74-5ffb-4cbc-b119-b79f2db00df3: !Template + answer_choices: Yes ||| No + id: 8f354d74-5ffb-4cbc-b119-b79f2db00df3 + jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' + a1997343-0a20-4f8c-8939-a512f0f68485: !Template + answer_choices: Yes ||| No + id: a1997343-0a20-4f8c-8939-a512f0f68485 + jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' + c45d5bac-968e-4e39-96b7-d18a5ddb225c: !Template + answer_choices: A ||| B + id: c45d5bac-968e-4e39-96b7-d18a5ddb225c + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'A/B choice 1 (good-bad, answer: A)' + reference: '' \ No newline at end of file From de16aa38922d9ecc1ec4d543de0603bb4122b09f Mon Sep 17 00:00:00 2001 From: Urmish Date: Mon, 25 Apr 2022 17:41:59 -0500 Subject: [PATCH 06/16] Added prompts for complex_NP_island --- .../blimp/complex_NP_island/templates.yaml | 165 ++++++++++++++++++ 1 file changed, 165 insertions(+) create mode 100644 promptsource/templates/blimp/complex_NP_island/templates.yaml diff --git a/promptsource/templates/blimp/complex_NP_island/templates.yaml b/promptsource/templates/blimp/complex_NP_island/templates.yaml new file mode 100644 index 000000000..f50482674 --- /dev/null +++ b/promptsource/templates/blimp/complex_NP_island/templates.yaml @@ -0,0 +1,165 @@ +dataset: blimp +subset: complex_NP_island +templates: + 0e5b5e79-cdea-47d3-b31c-c061162d3203: !Template + answer_choices: null + id: 0e5b5e79-cdea-47d3-b31c-c061162d3203 + jinja: 'Between the first sentence and the second sentence, which one is grammatically + correct? + + Sentence 1 - {{sentence_good}} + + Sentence 2 - {{sentence_bad}} + + + Answer - ||| + + Sentence 1' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: good_first_1 + reference: '' + 2599b172-d01c-4945-8ad2-5100c5eece1a: !Template + answer_choices: null + id: 2599b172-d01c-4945-8ad2-5100c5eece1a + jinja: 'Amongst the two sentences presented below, which one is syntactically + correct? + + + Sentence 1 - {{sentence_bad}} + + Sentence 2 - {{sentence_good}} + + + Answer - ||| + + Sentence 2' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: bad_first_2 + reference: '' + 7e8b5aa4-be3d-48f1-ba58-ab9eea3c7fc0: !Template + answer_choices: null + id: 7e8b5aa4-be3d-48f1-ba58-ab9eea3c7fc0 + jinja: 'Between the first sentence and the second sentence, which one is grammatically + correct? + + Sentence 1 - {{sentence_bad}} + + Sentence 2 - {{sentence_good}} + + + Answer - ||| + + Sentence 2' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: bad_first_1 + reference: '' + d6195bf0-b9df-456f-b66c-f22fc150aedd: !Template + answer_choices: null + id: d6195bf0-b9df-456f-b66c-f22fc150aedd + jinja: 'Amongst the two sentences presented below, which one is syntactically + correct? + + + Sentence 1 - {{sentence_good}} + + Sentence 2 - {{sentence_bad}} + + + Answer - ||| + + Sentence 1' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: good_first_2 + reference: '' + 3d2821a7-1c9c-44dd-ad23-e964b0998d1b: !Template + answer_choices: A ||| B + id: 3d2821a7-1c9c-44dd-ad23-e964b0998d1b + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'A/B choice 2 (bad-good, answer: B)' + reference: '' + 41594c98-baea-4b1f-a631-27842cd363fd: !Template + answer_choices: B ||| A + id: 41594c98-baea-4b1f-a631-27842cd363fd + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'B/A choice 1 (good-bad, answer: A)' + reference: '' + 88abc121-43ea-4b67-92e3-d762884d391d: !Template + answer_choices: B ||| A + id: 88abc121-43ea-4b67-92e3-d762884d391d + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'B/A choice, 2 (bad-good, answer: B)' + reference: '' + 8f354d74-5ffb-4cbc-b119-b79f2db00df3: !Template + answer_choices: Yes ||| No + id: 8f354d74-5ffb-4cbc-b119-b79f2db00df3 + jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' + a1997343-0a20-4f8c-8939-a512f0f68485: !Template + answer_choices: Yes ||| No + id: a1997343-0a20-4f8c-8939-a512f0f68485 + jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' + c45d5bac-968e-4e39-96b7-d18a5ddb225c: !Template + answer_choices: A ||| B + id: c45d5bac-968e-4e39-96b7-d18a5ddb225c + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'A/B choice 1 (good-bad, answer: A)' + reference: '' From ef823c4188d007bb6192867010153c5863fab7a6 Mon Sep 17 00:00:00 2001 From: Urmish Date: Mon, 25 Apr 2022 18:48:17 -0500 Subject: [PATCH 07/16] Updated the templates after the requested fixes --- .../blimp/adjunct_island/templates.yaml | 226 ++++++++++++------ .../blimp/complex_NP_island/templates.yaml | 165 ------------- 2 files changed, 148 insertions(+), 243 deletions(-) delete mode 100644 promptsource/templates/blimp/complex_NP_island/templates.yaml diff --git a/promptsource/templates/blimp/adjunct_island/templates.yaml b/promptsource/templates/blimp/adjunct_island/templates.yaml index 4013f34be..e3b77921b 100644 --- a/promptsource/templates/blimp/adjunct_island/templates.yaml +++ b/promptsource/templates/blimp/adjunct_island/templates.yaml @@ -1,117 +1,109 @@ dataset: blimp subset: adjunct_island templates: - 0e5b5e79-cdea-47d3-b31c-c061162d3203: !Template - answer_choices: null - id: 0e5b5e79-cdea-47d3-b31c-c061162d3203 - jinja: 'Between the first sentence and the second sentence, which one is grammatically - correct? - - Sentence 1 - {{sentence_good}} - - Sentence 2 - {{sentence_bad}} - - - Answer - ||| - - Sentence 1' + 3d2821a7-1c9c-44dd-ad23-e964b0998d1b: !Template + answer_choices: A ||| B + id: 3d2821a7-1c9c-44dd-ad23-e964b0998d1b + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] + }}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: adjunct_good_first_1 + name: 'A/B choice 2 (bad-good, answer: B)' reference: '' - 2599b172-d01c-4945-8ad2-5100c5eece1a: !Template - answer_choices: null - id: 2599b172-d01c-4945-8ad2-5100c5eece1a - jinja: 'Amongst the two sentences presented below, which one is syntactically - correct? - - - Sentence 1 - {{sentence_bad}} - - Sentence 2 - {{sentence_good}} - - - Answer - ||| - - Sentence 2' + 41594c98-baea-4b1f-a631-27842cd363fd: !Template + answer_choices: B ||| A + id: 41594c98-baea-4b1f-a631-27842cd363fd + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] + }}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: adjunct_bad_first_2 + name: 'B/A choice 1 (good-bad, answer: A)' reference: '' - 7e8b5aa4-be3d-48f1-ba58-ab9eea3c7fc0: !Template + 6e71cd4b-882c-4458-9cd4-fd83f849ad09: !Template answer_choices: null - id: 7e8b5aa4-be3d-48f1-ba58-ab9eea3c7fc0 - jinja: 'Between the first sentence and the second sentence, which one is grammatically - correct? + id: 6e71cd4b-882c-4458-9cd4-fd83f849ad09 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} - Sentence 1 - {{sentence_bad}} + 2: {{ sentence_bad }} - Sentence 2 - {{sentence_good}} + {% else %} + 1: {{ sentence_bad }} - Answer - ||| + 2: {{ sentence_good }} - Sentence 2' + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: adjunct_bad_first_1 + name: random_choice_1_2_2 reference: '' - d6195bf0-b9df-456f-b66c-f22fc150aedd: !Template + 7ed733fd-524c-4f00-800f-77a7a862ab4a: !Template answer_choices: null - id: d6195bf0-b9df-456f-b66c-f22fc150aedd - jinja: 'Amongst the two sentences presented below, which one is syntactically - correct? + id: 7ed733fd-524c-4f00-800f-77a7a862ab4a + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + 1: {{ sentence_good }} - Sentence 1 - {{sentence_good}} + 2: {{ sentence_bad }} - Sentence 2 - {{sentence_bad}} + {% else %} + 1: {{ sentence_bad }} - Answer - ||| + 2: {{ sentence_good }} - Sentence 1' + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: adjunct_good_first_2 - reference: '' - 3d2821a7-1c9c-44dd-ad23-e964b0998d1b: !Template - answer_choices: A ||| B - id: 3d2821a7-1c9c-44dd-ad23-e964b0998d1b - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'A/B choice 2 (bad-good, answer: B)' - reference: '' - 41594c98-baea-4b1f-a631-27842cd363fd: !Template - answer_choices: B ||| A - id: 41594c98-baea-4b1f-a631-27842cd363fd - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'B/A choice 1 (good-bad, answer: A)' + name: random_choice_1_2 reference: '' 88abc121-43ea-4b67-92e3-d762884d391d: !Template answer_choices: B ||| A @@ -123,7 +115,7 @@ templates: choices_in_prompt: true metrics: - Accuracy - original_task: false + original_task: true name: 'B/A choice, 2 (bad-good, answer: B)' reference: '' 8f354d74-5ffb-4cbc-b119-b79f2db00df3: !Template @@ -138,6 +130,45 @@ templates: original_task: false name: Yes/No binary judgment for bad sentences reference: '' + 967ca4e2-162f-4365-a1f5-2c87c0e874ce: !Template + answer_choices: null + id: 967ca4e2-162f-4365-a1f5-2c87c0e874ce + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B_2 + reference: '' a1997343-0a20-4f8c-8939-a512f0f68485: !Template answer_choices: Yes ||| No id: a1997343-0a20-4f8c-8939-a512f0f68485 @@ -160,6 +191,45 @@ templates: choices_in_prompt: true metrics: - Accuracy - original_task: false + original_task: true name: 'A/B choice 1 (good-bad, answer: A)' - reference: '' \ No newline at end of file + reference: '' + d11a37d7-7c42-4ee0-96c5-ffd24ad7073e: !Template + answer_choices: null + id: d11a37d7-7c42-4ee0-96c5-ffd24ad7073e + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B + reference: '' diff --git a/promptsource/templates/blimp/complex_NP_island/templates.yaml b/promptsource/templates/blimp/complex_NP_island/templates.yaml deleted file mode 100644 index f50482674..000000000 --- a/promptsource/templates/blimp/complex_NP_island/templates.yaml +++ /dev/null @@ -1,165 +0,0 @@ -dataset: blimp -subset: complex_NP_island -templates: - 0e5b5e79-cdea-47d3-b31c-c061162d3203: !Template - answer_choices: null - id: 0e5b5e79-cdea-47d3-b31c-c061162d3203 - jinja: 'Between the first sentence and the second sentence, which one is grammatically - correct? - - Sentence 1 - {{sentence_good}} - - Sentence 2 - {{sentence_bad}} - - - Answer - ||| - - Sentence 1' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: good_first_1 - reference: '' - 2599b172-d01c-4945-8ad2-5100c5eece1a: !Template - answer_choices: null - id: 2599b172-d01c-4945-8ad2-5100c5eece1a - jinja: 'Amongst the two sentences presented below, which one is syntactically - correct? - - - Sentence 1 - {{sentence_bad}} - - Sentence 2 - {{sentence_good}} - - - Answer - ||| - - Sentence 2' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: bad_first_2 - reference: '' - 7e8b5aa4-be3d-48f1-ba58-ab9eea3c7fc0: !Template - answer_choices: null - id: 7e8b5aa4-be3d-48f1-ba58-ab9eea3c7fc0 - jinja: 'Between the first sentence and the second sentence, which one is grammatically - correct? - - Sentence 1 - {{sentence_bad}} - - Sentence 2 - {{sentence_good}} - - - Answer - ||| - - Sentence 2' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: bad_first_1 - reference: '' - d6195bf0-b9df-456f-b66c-f22fc150aedd: !Template - answer_choices: null - id: d6195bf0-b9df-456f-b66c-f22fc150aedd - jinja: 'Amongst the two sentences presented below, which one is syntactically - correct? - - - Sentence 1 - {{sentence_good}} - - Sentence 2 - {{sentence_bad}} - - - Answer - ||| - - Sentence 1' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: good_first_2 - reference: '' - 3d2821a7-1c9c-44dd-ad23-e964b0998d1b: !Template - answer_choices: A ||| B - id: 3d2821a7-1c9c-44dd-ad23-e964b0998d1b - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'A/B choice 2 (bad-good, answer: B)' - reference: '' - 41594c98-baea-4b1f-a631-27842cd363fd: !Template - answer_choices: B ||| A - id: 41594c98-baea-4b1f-a631-27842cd363fd - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'B/A choice 1 (good-bad, answer: A)' - reference: '' - 88abc121-43ea-4b67-92e3-d762884d391d: !Template - answer_choices: B ||| A - id: 88abc121-43ea-4b67-92e3-d762884d391d - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'B/A choice, 2 (bad-good, answer: B)' - reference: '' - 8f354d74-5ffb-4cbc-b119-b79f2db00df3: !Template - answer_choices: Yes ||| No - id: 8f354d74-5ffb-4cbc-b119-b79f2db00df3 - jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] - }} - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences - reference: '' - a1997343-0a20-4f8c-8939-a512f0f68485: !Template - answer_choices: Yes ||| No - id: a1997343-0a20-4f8c-8939-a512f0f68485 - jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] - }} - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences - reference: '' - c45d5bac-968e-4e39-96b7-d18a5ddb225c: !Template - answer_choices: A ||| B - id: c45d5bac-968e-4e39-96b7-d18a5ddb225c - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'A/B choice 1 (good-bad, answer: A)' - reference: '' From 1cd898ecf1618b29556da3ed87ade3376152ad99 Mon Sep 17 00:00:00 2001 From: Urmish Date: Mon, 25 Apr 2022 19:00:36 -0500 Subject: [PATCH 08/16] Added templates for complex_NP_island via copy scripts --- .../blimp/complex_NP_island/templates.yaml | 235 ++++++++++++++++++ 1 file changed, 235 insertions(+) create mode 100644 promptsource/templates/blimp/complex_NP_island/templates.yaml diff --git a/promptsource/templates/blimp/complex_NP_island/templates.yaml b/promptsource/templates/blimp/complex_NP_island/templates.yaml new file mode 100644 index 000000000..88f3d8e2c --- /dev/null +++ b/promptsource/templates/blimp/complex_NP_island/templates.yaml @@ -0,0 +1,235 @@ +dataset: blimp +subset: complex_NP_island +templates: + 2607ed18-bd42-4f19-bbaf-a23b7558e7e4: !Template + answer_choices: B ||| A + id: 2607ed18-bd42-4f19-bbaf-a23b7558e7e4 + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice 1 (good-bad, answer: A)' + reference: '' + 2cb9d1cf-0867-4c17-a845-d5737dcf2e94: !Template + answer_choices: null + id: 2cb9d1cf-0867-4c17-a845-d5737dcf2e94 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B + reference: '' + 6b647b69-c034-4f5d-aaec-0997f60f456e: !Template + answer_choices: A ||| B + id: 6b647b69-c034-4f5d-aaec-0997f60f456e + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 2 (bad-good, answer: B)' + reference: '' + 6caa4ee8-bb37-46bc-b4e2-f7af9f5f7120: !Template + answer_choices: null + id: 6caa4ee8-bb37-46bc-b4e2-f7af9f5f7120 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2_2 + reference: '' + 7a2a5800-3ea2-4171-952d-8351a6cee375: !Template + answer_choices: A ||| B + id: 7a2a5800-3ea2-4171-952d-8351a6cee375 + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 1 (good-bad, answer: A)' + reference: '' + 88626f98-028c-472c-9f82-00a9b99c36dd: !Template + answer_choices: null + id: 88626f98-028c-472c-9f82-00a9b99c36dd + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2 + reference: '' + 8e6c7628-a338-4b21-ab0e-7d3004596674: !Template + answer_choices: B ||| A + id: 8e6c7628-a338-4b21-ab0e-7d3004596674 + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice, 2 (bad-good, answer: B)' + reference: '' + b9767247-12a9-4889-8009-eb87f59b7f77: !Template + answer_choices: Yes ||| No + id: b9767247-12a9-4889-8009-eb87f59b7f77 + jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' + d52411a5-e1b0-4ef4-ae59-7e470884477d: !Template + answer_choices: null + id: d52411a5-e1b0-4ef4-ae59-7e470884477d + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B_2 + reference: '' + d6d4a7f1-db0e-441b-8f6c-33333926e630: !Template + answer_choices: Yes ||| No + id: d6d4a7f1-db0e-441b-8f6c-33333926e630 + jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' From a43174915f535e25d08639c591ddc65f57ee301a Mon Sep 17 00:00:00 2001 From: Urmish Date: Mon, 25 Apr 2022 19:22:23 -0500 Subject: [PATCH 09/16] Updated templates for all subsets --- .../anaphor_gender_agreement/templates.yaml | 235 ++++++++++++++++++ .../anaphor_number_agreement/templates.yaml | 235 ++++++++++++++++++ .../animate_subject_passive/templates.yaml | 235 ++++++++++++++++++ .../animate_subject_trans/templates.yaml | 235 ++++++++++++++++++ .../templates/blimp/causative/templates.yaml | 235 ++++++++++++++++++ .../templates.yaml | 235 ++++++++++++++++++ .../templates.yaml | 235 ++++++++++++++++++ .../templates.yaml | 235 ++++++++++++++++++ .../templates.yaml | 235 ++++++++++++++++++ .../templates.yaml | 235 ++++++++++++++++++ .../templates.yaml | 235 ++++++++++++++++++ .../templates.yaml | 235 ++++++++++++++++++ .../templates.yaml | 235 ++++++++++++++++++ .../templates.yaml | 235 ++++++++++++++++++ .../templates.yaml | 235 ++++++++++++++++++ .../templates.yaml | 235 ++++++++++++++++++ .../templates.yaml | 235 ++++++++++++++++++ .../blimp/drop_argument/templates.yaml | 235 ++++++++++++++++++ .../blimp/ellipsis_n_bar_1/templates.yaml | 235 ++++++++++++++++++ .../blimp/ellipsis_n_bar_2/templates.yaml | 235 ++++++++++++++++++ .../templates.yaml | 235 ++++++++++++++++++ .../templates.yaml | 235 ++++++++++++++++++ .../templates.yaml | 235 ++++++++++++++++++ .../templates.yaml | 235 ++++++++++++++++++ .../templates.yaml | 235 ++++++++++++++++++ .../templates/blimp/inchoative/templates.yaml | 235 ++++++++++++++++++ .../blimp/intransitive/templates.yaml | 235 ++++++++++++++++++ .../templates.yaml | 235 ++++++++++++++++++ .../templates.yaml | 235 ++++++++++++++++++ .../templates.yaml | 235 ++++++++++++++++++ .../templates.yaml | 235 ++++++++++++++++++ .../templates.yaml | 235 ++++++++++++++++++ .../templates.yaml | 235 ++++++++++++++++++ .../templates.yaml | 235 ++++++++++++++++++ .../blimp/npi_present_1/templates.yaml | 235 ++++++++++++++++++ .../blimp/npi_present_2/templates.yaml | 235 ++++++++++++++++++ .../only_npi_licensor_present/templates.yaml | 235 ++++++++++++++++++ .../blimp/only_npi_scope/templates.yaml | 235 ++++++++++++++++++ .../templates/blimp/passive_1/templates.yaml | 235 ++++++++++++++++++ .../templates/blimp/passive_2/templates.yaml | 235 ++++++++++++++++++ .../principle_A_c_command/templates.yaml | 235 ++++++++++++++++++ .../blimp/principle_A_case_1/templates.yaml | 235 ++++++++++++++++++ .../blimp/principle_A_case_2/templates.yaml | 235 ++++++++++++++++++ .../blimp/principle_A_domain_1/templates.yaml | 235 ++++++++++++++++++ .../blimp/principle_A_domain_2/templates.yaml | 235 ++++++++++++++++++ .../blimp/principle_A_domain_3/templates.yaml | 235 ++++++++++++++++++ .../principle_A_reconstruction/templates.yaml | 235 ++++++++++++++++++ .../templates.yaml | 235 ++++++++++++++++++ .../templates.yaml | 235 ++++++++++++++++++ .../templates.yaml | 235 ++++++++++++++++++ .../templates.yaml | 235 ++++++++++++++++++ .../sentential_subject_island/templates.yaml | 235 ++++++++++++++++++ .../superlative_quantifiers_1/templates.yaml | 235 ++++++++++++++++++ .../superlative_quantifiers_2/templates.yaml | 235 ++++++++++++++++++ .../blimp/tough_vs_raising_1/templates.yaml | 235 ++++++++++++++++++ .../blimp/tough_vs_raising_2/templates.yaml | 235 ++++++++++++++++++ .../templates/blimp/transitive/templates.yaml | 235 ++++++++++++++++++ .../templates/blimp/wh_island/templates.yaml | 235 ++++++++++++++++++ .../wh_questions_object_gap/templates.yaml | 235 ++++++++++++++++++ .../wh_questions_subject_gap/templates.yaml | 235 ++++++++++++++++++ .../templates.yaml | 235 ++++++++++++++++++ .../blimp/wh_vs_that_no_gap/templates.yaml | 235 ++++++++++++++++++ .../templates.yaml | 235 ++++++++++++++++++ .../blimp/wh_vs_that_with_gap/templates.yaml | 235 ++++++++++++++++++ .../templates.yaml | 235 ++++++++++++++++++ 65 files changed, 15275 insertions(+) create mode 100644 promptsource/templates/blimp/anaphor_gender_agreement/templates.yaml create mode 100644 promptsource/templates/blimp/anaphor_number_agreement/templates.yaml create mode 100644 promptsource/templates/blimp/animate_subject_passive/templates.yaml create mode 100644 promptsource/templates/blimp/animate_subject_trans/templates.yaml create mode 100644 promptsource/templates/blimp/causative/templates.yaml create mode 100644 promptsource/templates/blimp/coordinate_structure_constraint_complex_left_branch/templates.yaml create mode 100644 promptsource/templates/blimp/coordinate_structure_constraint_object_extraction/templates.yaml create mode 100644 promptsource/templates/blimp/determiner_noun_agreement_1/templates.yaml create mode 100644 promptsource/templates/blimp/determiner_noun_agreement_2/templates.yaml create mode 100644 promptsource/templates/blimp/determiner_noun_agreement_irregular_1/templates.yaml create mode 100644 promptsource/templates/blimp/determiner_noun_agreement_irregular_2/templates.yaml create mode 100644 promptsource/templates/blimp/determiner_noun_agreement_with_adj_2/templates.yaml create mode 100644 promptsource/templates/blimp/determiner_noun_agreement_with_adj_irregular_1/templates.yaml create mode 100644 promptsource/templates/blimp/determiner_noun_agreement_with_adj_irregular_2/templates.yaml create mode 100644 promptsource/templates/blimp/determiner_noun_agreement_with_adjective_1/templates.yaml create mode 100644 promptsource/templates/blimp/distractor_agreement_relational_noun/templates.yaml create mode 100644 promptsource/templates/blimp/distractor_agreement_relative_clause/templates.yaml create mode 100644 promptsource/templates/blimp/drop_argument/templates.yaml create mode 100644 promptsource/templates/blimp/ellipsis_n_bar_1/templates.yaml create mode 100644 promptsource/templates/blimp/ellipsis_n_bar_2/templates.yaml create mode 100644 promptsource/templates/blimp/existential_there_object_raising/templates.yaml create mode 100644 promptsource/templates/blimp/existential_there_quantifiers_1/templates.yaml create mode 100644 promptsource/templates/blimp/existential_there_quantifiers_2/templates.yaml create mode 100644 promptsource/templates/blimp/existential_there_subject_raising/templates.yaml create mode 100644 promptsource/templates/blimp/expletive_it_object_raising/templates.yaml create mode 100644 promptsource/templates/blimp/inchoative/templates.yaml create mode 100644 promptsource/templates/blimp/intransitive/templates.yaml create mode 100644 promptsource/templates/blimp/irregular_past_participle_adjectives/templates.yaml create mode 100644 promptsource/templates/blimp/irregular_past_participle_verbs/templates.yaml create mode 100644 promptsource/templates/blimp/irregular_plural_subject_verb_agreement_1/templates.yaml create mode 100644 promptsource/templates/blimp/irregular_plural_subject_verb_agreement_2/templates.yaml create mode 100644 promptsource/templates/blimp/left_branch_island_echo_question/templates.yaml create mode 100644 promptsource/templates/blimp/left_branch_island_simple_question/templates.yaml create mode 100644 promptsource/templates/blimp/matrix_question_npi_licensor_present/templates.yaml create mode 100644 promptsource/templates/blimp/npi_present_1/templates.yaml create mode 100644 promptsource/templates/blimp/npi_present_2/templates.yaml create mode 100644 promptsource/templates/blimp/only_npi_licensor_present/templates.yaml create mode 100644 promptsource/templates/blimp/only_npi_scope/templates.yaml create mode 100644 promptsource/templates/blimp/passive_1/templates.yaml create mode 100644 promptsource/templates/blimp/passive_2/templates.yaml create mode 100644 promptsource/templates/blimp/principle_A_c_command/templates.yaml create mode 100644 promptsource/templates/blimp/principle_A_case_1/templates.yaml create mode 100644 promptsource/templates/blimp/principle_A_case_2/templates.yaml create mode 100644 promptsource/templates/blimp/principle_A_domain_1/templates.yaml create mode 100644 promptsource/templates/blimp/principle_A_domain_2/templates.yaml create mode 100644 promptsource/templates/blimp/principle_A_domain_3/templates.yaml create mode 100644 promptsource/templates/blimp/principle_A_reconstruction/templates.yaml create mode 100644 promptsource/templates/blimp/regular_plural_subject_verb_agreement_1/templates.yaml create mode 100644 promptsource/templates/blimp/regular_plural_subject_verb_agreement_2/templates.yaml create mode 100644 promptsource/templates/blimp/sentential_negation_npi_licensor_present/templates.yaml create mode 100644 promptsource/templates/blimp/sentential_negation_npi_scope/templates.yaml create mode 100644 promptsource/templates/blimp/sentential_subject_island/templates.yaml create mode 100644 promptsource/templates/blimp/superlative_quantifiers_1/templates.yaml create mode 100644 promptsource/templates/blimp/superlative_quantifiers_2/templates.yaml create mode 100644 promptsource/templates/blimp/tough_vs_raising_1/templates.yaml create mode 100644 promptsource/templates/blimp/tough_vs_raising_2/templates.yaml create mode 100644 promptsource/templates/blimp/transitive/templates.yaml create mode 100644 promptsource/templates/blimp/wh_island/templates.yaml create mode 100644 promptsource/templates/blimp/wh_questions_object_gap/templates.yaml create mode 100644 promptsource/templates/blimp/wh_questions_subject_gap/templates.yaml create mode 100644 promptsource/templates/blimp/wh_questions_subject_gap_long_distance/templates.yaml create mode 100644 promptsource/templates/blimp/wh_vs_that_no_gap/templates.yaml create mode 100644 promptsource/templates/blimp/wh_vs_that_no_gap_long_distance/templates.yaml create mode 100644 promptsource/templates/blimp/wh_vs_that_with_gap/templates.yaml create mode 100644 promptsource/templates/blimp/wh_vs_that_with_gap_long_distance/templates.yaml diff --git a/promptsource/templates/blimp/anaphor_gender_agreement/templates.yaml b/promptsource/templates/blimp/anaphor_gender_agreement/templates.yaml new file mode 100644 index 000000000..c4c6c4b00 --- /dev/null +++ b/promptsource/templates/blimp/anaphor_gender_agreement/templates.yaml @@ -0,0 +1,235 @@ +dataset: blimp +subset: anaphor_gender_agreement +templates: + 1a9f6ffa-2f8a-497f-8027-83247e8a3db7: !Template + answer_choices: null + id: 1a9f6ffa-2f8a-497f-8027-83247e8a3db7 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2_2 + reference: '' + 26edb4c7-2248-4f0e-af34-a4316bd7bb15: !Template + answer_choices: Yes ||| No + id: 26edb4c7-2248-4f0e-af34-a4316bd7bb15 + jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' + 440c3a37-3df7-456e-8f8f-68953ce605e8: !Template + answer_choices: null + id: 440c3a37-3df7-456e-8f8f-68953ce605e8 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B + reference: '' + 4c2f4b2f-542e-4d04-8a5c-9a1fedfc4079: !Template + answer_choices: null + id: 4c2f4b2f-542e-4d04-8a5c-9a1fedfc4079 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B_2 + reference: '' + 5fc6b5cf-e083-485b-b727-7b1daf182102: !Template + answer_choices: A ||| B + id: 5fc6b5cf-e083-485b-b727-7b1daf182102 + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 2 (bad-good, answer: B)' + reference: '' + 6de15abd-3a32-42b0-a2e5-997487ef9f81: !Template + answer_choices: A ||| B + id: 6de15abd-3a32-42b0-a2e5-997487ef9f81 + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 1 (good-bad, answer: A)' + reference: '' + 710fe183-95cc-4228-b3c5-a141f0f56da1: !Template + answer_choices: B ||| A + id: 710fe183-95cc-4228-b3c5-a141f0f56da1 + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice, 2 (bad-good, answer: B)' + reference: '' + 9fa6e2d6-430d-4f5e-b54f-7e1c237b2557: !Template + answer_choices: null + id: 9fa6e2d6-430d-4f5e-b54f-7e1c237b2557 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2 + reference: '' + af619299-690d-41a3-96da-c2369d152dbf: !Template + answer_choices: B ||| A + id: af619299-690d-41a3-96da-c2369d152dbf + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice 1 (good-bad, answer: A)' + reference: '' + e2f4b270-eac8-4649-9f34-1090186d2487: !Template + answer_choices: Yes ||| No + id: e2f4b270-eac8-4649-9f34-1090186d2487 + jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' diff --git a/promptsource/templates/blimp/anaphor_number_agreement/templates.yaml b/promptsource/templates/blimp/anaphor_number_agreement/templates.yaml new file mode 100644 index 000000000..a6b6aa006 --- /dev/null +++ b/promptsource/templates/blimp/anaphor_number_agreement/templates.yaml @@ -0,0 +1,235 @@ +dataset: blimp +subset: anaphor_number_agreement +templates: + 15858e60-b678-4ae7-b5a4-98b792b95dc3: !Template + answer_choices: A ||| B + id: 15858e60-b678-4ae7-b5a4-98b792b95dc3 + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 1 (good-bad, answer: A)' + reference: '' + 5396f9fa-2712-4af7-bdb0-68be1253c2d8: !Template + answer_choices: null + id: 5396f9fa-2712-4af7-bdb0-68be1253c2d8 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2 + reference: '' + 7a8b9db6-969e-4e0a-8abd-b270cb414268: !Template + answer_choices: Yes ||| No + id: 7a8b9db6-969e-4e0a-8abd-b270cb414268 + jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' + 8ea37f01-e441-4b7b-9652-94ed6990f9ac: !Template + answer_choices: B ||| A + id: 8ea37f01-e441-4b7b-9652-94ed6990f9ac + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice 1 (good-bad, answer: A)' + reference: '' + 993edeef-84eb-4c7d-92bc-c74187ea9054: !Template + answer_choices: A ||| B + id: 993edeef-84eb-4c7d-92bc-c74187ea9054 + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 2 (bad-good, answer: B)' + reference: '' + a0dc89be-5646-45de-a6bd-6c54a88be38e: !Template + answer_choices: null + id: a0dc89be-5646-45de-a6bd-6c54a88be38e + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B + reference: '' + c638fc37-0a6f-408f-bca6-98f6fa29c989: !Template + answer_choices: Yes ||| No + id: c638fc37-0a6f-408f-bca6-98f6fa29c989 + jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' + c8d6f39d-ad35-43ad-88bc-a2eb2757598c: !Template + answer_choices: B ||| A + id: c8d6f39d-ad35-43ad-88bc-a2eb2757598c + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice, 2 (bad-good, answer: B)' + reference: '' + e6ab9771-3180-42bd-b374-d40a5ebb85c7: !Template + answer_choices: null + id: e6ab9771-3180-42bd-b374-d40a5ebb85c7 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2_2 + reference: '' + f587556a-8f43-4ff6-975b-1c5d625edf6c: !Template + answer_choices: null + id: f587556a-8f43-4ff6-975b-1c5d625edf6c + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B_2 + reference: '' diff --git a/promptsource/templates/blimp/animate_subject_passive/templates.yaml b/promptsource/templates/blimp/animate_subject_passive/templates.yaml new file mode 100644 index 000000000..a4a0f9da8 --- /dev/null +++ b/promptsource/templates/blimp/animate_subject_passive/templates.yaml @@ -0,0 +1,235 @@ +dataset: blimp +subset: animate_subject_passive +templates: + 2aa95c35-d9c6-4bcb-bc3e-4e9add599cc3: !Template + answer_choices: A ||| B + id: 2aa95c35-d9c6-4bcb-bc3e-4e9add599cc3 + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 1 (good-bad, answer: A)' + reference: '' + 5a8a4633-d508-4f34-a4d5-8fc0e46a8ab4: !Template + answer_choices: B ||| A + id: 5a8a4633-d508-4f34-a4d5-8fc0e46a8ab4 + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice 1 (good-bad, answer: A)' + reference: '' + 5c5ded69-0492-4495-8290-ada132a6161c: !Template + answer_choices: Yes ||| No + id: 5c5ded69-0492-4495-8290-ada132a6161c + jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' + a90f7a73-56b8-4bc0-bc7b-5ee760578101: !Template + answer_choices: null + id: a90f7a73-56b8-4bc0-bc7b-5ee760578101 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2_2 + reference: '' + bc68fae5-2635-4aef-9d7a-f931877f1c2d: !Template + answer_choices: Yes ||| No + id: bc68fae5-2635-4aef-9d7a-f931877f1c2d + jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' + bdb9186b-4c0e-41d2-9ba7-8f6cc824d90f: !Template + answer_choices: null + id: bdb9186b-4c0e-41d2-9ba7-8f6cc824d90f + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B + reference: '' + c36492ef-9b69-4b4e-8332-1b5983fff753: !Template + answer_choices: null + id: c36492ef-9b69-4b4e-8332-1b5983fff753 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2 + reference: '' + e6b8a2a5-f397-453c-83da-aa73d9906423: !Template + answer_choices: A ||| B + id: e6b8a2a5-f397-453c-83da-aa73d9906423 + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 2 (bad-good, answer: B)' + reference: '' + e71bc916-ed97-4a14-8eb4-2871c7b5267a: !Template + answer_choices: B ||| A + id: e71bc916-ed97-4a14-8eb4-2871c7b5267a + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice, 2 (bad-good, answer: B)' + reference: '' + e7d26950-94eb-4bf1-b7b2-958e9f7ca555: !Template + answer_choices: null + id: e7d26950-94eb-4bf1-b7b2-958e9f7ca555 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B_2 + reference: '' diff --git a/promptsource/templates/blimp/animate_subject_trans/templates.yaml b/promptsource/templates/blimp/animate_subject_trans/templates.yaml new file mode 100644 index 000000000..1b4b8132f --- /dev/null +++ b/promptsource/templates/blimp/animate_subject_trans/templates.yaml @@ -0,0 +1,235 @@ +dataset: blimp +subset: animate_subject_trans +templates: + 2a836319-30fa-4ee4-b04c-2fdd21fabf8d: !Template + answer_choices: A ||| B + id: 2a836319-30fa-4ee4-b04c-2fdd21fabf8d + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 2 (bad-good, answer: B)' + reference: '' + 8eb6ce9f-6878-48e8-b210-8310fa11fb80: !Template + answer_choices: null + id: 8eb6ce9f-6878-48e8-b210-8310fa11fb80 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2 + reference: '' + a4b6ff3c-a6f9-49c6-9e5e-dee4d1505d25: !Template + answer_choices: null + id: a4b6ff3c-a6f9-49c6-9e5e-dee4d1505d25 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B + reference: '' + aae78eb4-d350-4feb-9745-84b1b6d06754: !Template + answer_choices: null + id: aae78eb4-d350-4feb-9745-84b1b6d06754 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B_2 + reference: '' + baa9feb8-b4e6-4340-a1ef-3729f86a8739: !Template + answer_choices: Yes ||| No + id: baa9feb8-b4e6-4340-a1ef-3729f86a8739 + jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' + be885170-cf6d-4744-9801-6ed98c7f5c74: !Template + answer_choices: A ||| B + id: be885170-cf6d-4744-9801-6ed98c7f5c74 + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 1 (good-bad, answer: A)' + reference: '' + d8d53f3e-2c75-4b9f-835b-88a1b4b245f8: !Template + answer_choices: null + id: d8d53f3e-2c75-4b9f-835b-88a1b4b245f8 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2_2 + reference: '' + e67ca969-466d-413f-af05-1bb448391da8: !Template + answer_choices: B ||| A + id: e67ca969-466d-413f-af05-1bb448391da8 + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice 1 (good-bad, answer: A)' + reference: '' + eca41738-860d-40ad-b0a0-be7e5f6418dc: !Template + answer_choices: Yes ||| No + id: eca41738-860d-40ad-b0a0-be7e5f6418dc + jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' + f2c5595a-98ad-4d4b-b56a-1003f53865c8: !Template + answer_choices: B ||| A + id: f2c5595a-98ad-4d4b-b56a-1003f53865c8 + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice, 2 (bad-good, answer: B)' + reference: '' diff --git a/promptsource/templates/blimp/causative/templates.yaml b/promptsource/templates/blimp/causative/templates.yaml new file mode 100644 index 000000000..4911a649a --- /dev/null +++ b/promptsource/templates/blimp/causative/templates.yaml @@ -0,0 +1,235 @@ +dataset: blimp +subset: causative +templates: + 1052f205-1507-4e92-854c-720eac6eab0e: !Template + answer_choices: B ||| A + id: 1052f205-1507-4e92-854c-720eac6eab0e + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice, 2 (bad-good, answer: B)' + reference: '' + 3fe6c960-93b4-4754-a55c-df63c6ab420b: !Template + answer_choices: Yes ||| No + id: 3fe6c960-93b4-4754-a55c-df63c6ab420b + jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' + 4f9a24d2-b41e-409d-950b-0c16071b824c: !Template + answer_choices: A ||| B + id: 4f9a24d2-b41e-409d-950b-0c16071b824c + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 2 (bad-good, answer: B)' + reference: '' + 68ec16bd-a092-4925-b3df-4911a2f385c1: !Template + answer_choices: A ||| B + id: 68ec16bd-a092-4925-b3df-4911a2f385c1 + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 1 (good-bad, answer: A)' + reference: '' + 85053740-aceb-4df6-b5e5-8ee9d60d7abf: !Template + answer_choices: null + id: 85053740-aceb-4df6-b5e5-8ee9d60d7abf + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B_2 + reference: '' + 85cdc805-415e-462b-b582-847140fae237: !Template + answer_choices: Yes ||| No + id: 85cdc805-415e-462b-b582-847140fae237 + jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' + 93dd6231-2d92-43e4-9feb-fc1faec0b594: !Template + answer_choices: null + id: 93dd6231-2d92-43e4-9feb-fc1faec0b594 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B + reference: '' + b7fc3d16-37d1-4077-9909-98345754e7ef: !Template + answer_choices: null + id: b7fc3d16-37d1-4077-9909-98345754e7ef + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2_2 + reference: '' + cb4cede4-02f7-48ed-87d3-88ff805b8448: !Template + answer_choices: null + id: cb4cede4-02f7-48ed-87d3-88ff805b8448 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2 + reference: '' + e880242a-e482-4fd8-8108-a0882763ee99: !Template + answer_choices: B ||| A + id: e880242a-e482-4fd8-8108-a0882763ee99 + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice 1 (good-bad, answer: A)' + reference: '' diff --git a/promptsource/templates/blimp/coordinate_structure_constraint_complex_left_branch/templates.yaml b/promptsource/templates/blimp/coordinate_structure_constraint_complex_left_branch/templates.yaml new file mode 100644 index 000000000..886296625 --- /dev/null +++ b/promptsource/templates/blimp/coordinate_structure_constraint_complex_left_branch/templates.yaml @@ -0,0 +1,235 @@ +dataset: blimp +subset: coordinate_structure_constraint_complex_left_branch +templates: + 129e334d-48b1-4613-8b53-9be2dbe216a4: !Template + answer_choices: B ||| A + id: 129e334d-48b1-4613-8b53-9be2dbe216a4 + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice, 2 (bad-good, answer: B)' + reference: '' + 3422f883-6719-40be-aa3d-1ccb186db977: !Template + answer_choices: null + id: 3422f883-6719-40be-aa3d-1ccb186db977 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B + reference: '' + 3d6fb365-6d4e-4b4f-8ec4-389a2c621072: !Template + answer_choices: null + id: 3d6fb365-6d4e-4b4f-8ec4-389a2c621072 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2 + reference: '' + 3dbe2f72-0956-474f-a5c4-35ca59d6eb71: !Template + answer_choices: null + id: 3dbe2f72-0956-474f-a5c4-35ca59d6eb71 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2_2 + reference: '' + 440c43d8-bdca-490a-aca8-b10dba9e6ca1: !Template + answer_choices: Yes ||| No + id: 440c43d8-bdca-490a-aca8-b10dba9e6ca1 + jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' + 8e1b0404-e761-401a-b0b4-204a4c8018b2: !Template + answer_choices: A ||| B + id: 8e1b0404-e761-401a-b0b4-204a4c8018b2 + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 1 (good-bad, answer: A)' + reference: '' + 9f591c11-2ea8-4d4d-95df-f3cbae9f39af: !Template + answer_choices: A ||| B + id: 9f591c11-2ea8-4d4d-95df-f3cbae9f39af + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 2 (bad-good, answer: B)' + reference: '' + a61e0505-cf82-4239-855a-b79352b2ee94: !Template + answer_choices: null + id: a61e0505-cf82-4239-855a-b79352b2ee94 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B_2 + reference: '' + b90db41b-150f-422e-8bda-cfdab1443958: !Template + answer_choices: B ||| A + id: b90db41b-150f-422e-8bda-cfdab1443958 + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice 1 (good-bad, answer: A)' + reference: '' + ce927c2d-b220-459e-8382-44740d923591: !Template + answer_choices: Yes ||| No + id: ce927c2d-b220-459e-8382-44740d923591 + jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' diff --git a/promptsource/templates/blimp/coordinate_structure_constraint_object_extraction/templates.yaml b/promptsource/templates/blimp/coordinate_structure_constraint_object_extraction/templates.yaml new file mode 100644 index 000000000..667c9f0a6 --- /dev/null +++ b/promptsource/templates/blimp/coordinate_structure_constraint_object_extraction/templates.yaml @@ -0,0 +1,235 @@ +dataset: blimp +subset: coordinate_structure_constraint_object_extraction +templates: + 04122a9b-de72-44e0-9d5a-02e4a789c6ad: !Template + answer_choices: B ||| A + id: 04122a9b-de72-44e0-9d5a-02e4a789c6ad + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice 1 (good-bad, answer: A)' + reference: '' + 04581e94-4cc7-45ba-a986-01858160756c: !Template + answer_choices: null + id: 04581e94-4cc7-45ba-a986-01858160756c + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2 + reference: '' + 15f14103-d891-4f38-812f-ae0804deab11: !Template + answer_choices: Yes ||| No + id: 15f14103-d891-4f38-812f-ae0804deab11 + jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' + 221ccc0d-cfe8-4d0f-be60-42047cc0f71c: !Template + answer_choices: A ||| B + id: 221ccc0d-cfe8-4d0f-be60-42047cc0f71c + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 1 (good-bad, answer: A)' + reference: '' + 43be39a9-9bc2-4951-a377-d82ffa138780: !Template + answer_choices: B ||| A + id: 43be39a9-9bc2-4951-a377-d82ffa138780 + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice, 2 (bad-good, answer: B)' + reference: '' + 9097e81f-adfa-474d-a35d-e9faaa600559: !Template + answer_choices: Yes ||| No + id: 9097e81f-adfa-474d-a35d-e9faaa600559 + jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' + 9f1d7348-9bde-4083-812a-52567a95f2ea: !Template + answer_choices: null + id: 9f1d7348-9bde-4083-812a-52567a95f2ea + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2_2 + reference: '' + cc32ed2f-2c2f-476d-9fae-53efac1921f1: !Template + answer_choices: null + id: cc32ed2f-2c2f-476d-9fae-53efac1921f1 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B_2 + reference: '' + f54867ae-1231-4c3a-b7e9-f92ad2f4c2fe: !Template + answer_choices: A ||| B + id: f54867ae-1231-4c3a-b7e9-f92ad2f4c2fe + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 2 (bad-good, answer: B)' + reference: '' + f5f9f535-3082-4efe-9fa2-8c65ff8977c6: !Template + answer_choices: null + id: f5f9f535-3082-4efe-9fa2-8c65ff8977c6 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B + reference: '' diff --git a/promptsource/templates/blimp/determiner_noun_agreement_1/templates.yaml b/promptsource/templates/blimp/determiner_noun_agreement_1/templates.yaml new file mode 100644 index 000000000..aa0ec68fc --- /dev/null +++ b/promptsource/templates/blimp/determiner_noun_agreement_1/templates.yaml @@ -0,0 +1,235 @@ +dataset: blimp +subset: determiner_noun_agreement_1 +templates: + 0fd3119b-070a-4b76-93ed-45d1383dffd4: !Template + answer_choices: null + id: 0fd3119b-070a-4b76-93ed-45d1383dffd4 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2 + reference: '' + 194aa45a-aaf0-42be-8858-f2cb83c851ec: !Template + answer_choices: null + id: 194aa45a-aaf0-42be-8858-f2cb83c851ec + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2_2 + reference: '' + 280ae1d0-e23d-4b96-8fe6-cf7cba71f56a: !Template + answer_choices: Yes ||| No + id: 280ae1d0-e23d-4b96-8fe6-cf7cba71f56a + jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' + 44daca2a-6c53-4644-93fa-84b938e7ae58: !Template + answer_choices: A ||| B + id: 44daca2a-6c53-4644-93fa-84b938e7ae58 + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 1 (good-bad, answer: A)' + reference: '' + 657044d9-cb52-4c2b-b35c-423b5e3aaefc: !Template + answer_choices: null + id: 657044d9-cb52-4c2b-b35c-423b5e3aaefc + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B_2 + reference: '' + 665b5c4c-7919-4d2b-9b37-666e0631e04c: !Template + answer_choices: B ||| A + id: 665b5c4c-7919-4d2b-9b37-666e0631e04c + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice, 2 (bad-good, answer: B)' + reference: '' + 9f3e846a-217d-41a4-a6be-1fa0165996f4: !Template + answer_choices: null + id: 9f3e846a-217d-41a4-a6be-1fa0165996f4 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B + reference: '' + a3f470e9-2493-4a9c-8a9b-713b002e62ff: !Template + answer_choices: A ||| B + id: a3f470e9-2493-4a9c-8a9b-713b002e62ff + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 2 (bad-good, answer: B)' + reference: '' + cbbea246-35e8-4d7b-b640-26b18b9ac42c: !Template + answer_choices: B ||| A + id: cbbea246-35e8-4d7b-b640-26b18b9ac42c + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice 1 (good-bad, answer: A)' + reference: '' + df12f192-94b8-4d13-a129-da78e725adf5: !Template + answer_choices: Yes ||| No + id: df12f192-94b8-4d13-a129-da78e725adf5 + jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' diff --git a/promptsource/templates/blimp/determiner_noun_agreement_2/templates.yaml b/promptsource/templates/blimp/determiner_noun_agreement_2/templates.yaml new file mode 100644 index 000000000..9bc4229cb --- /dev/null +++ b/promptsource/templates/blimp/determiner_noun_agreement_2/templates.yaml @@ -0,0 +1,235 @@ +dataset: blimp +subset: determiner_noun_agreement_2 +templates: + 223e4cb1-3eec-47a0-846f-1b4cd02446b5: !Template + answer_choices: null + id: 223e4cb1-3eec-47a0-846f-1b4cd02446b5 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2_2 + reference: '' + 269cc713-a4bf-4031-bb61-4ba07df76199: !Template + answer_choices: null + id: 269cc713-a4bf-4031-bb61-4ba07df76199 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B_2 + reference: '' + 61d53280-467e-4d8a-a79d-ff5c4a5586dc: !Template + answer_choices: A ||| B + id: 61d53280-467e-4d8a-a79d-ff5c4a5586dc + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 2 (bad-good, answer: B)' + reference: '' + 633df725-56db-4a6e-9a2e-e352f08c7b9e: !Template + answer_choices: B ||| A + id: 633df725-56db-4a6e-9a2e-e352f08c7b9e + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice 1 (good-bad, answer: A)' + reference: '' + 974b3a6d-283f-4758-81f7-bfa5a75c4006: !Template + answer_choices: Yes ||| No + id: 974b3a6d-283f-4758-81f7-bfa5a75c4006 + jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' + 9be37f75-18d4-4a16-b214-36c7c6813380: !Template + answer_choices: B ||| A + id: 9be37f75-18d4-4a16-b214-36c7c6813380 + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice, 2 (bad-good, answer: B)' + reference: '' + 9dc0142e-b289-4fdd-a33e-67fbce65f7ee: !Template + answer_choices: Yes ||| No + id: 9dc0142e-b289-4fdd-a33e-67fbce65f7ee + jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' + cf8187e8-90ed-46fe-ba3e-68ed6f094547: !Template + answer_choices: null + id: cf8187e8-90ed-46fe-ba3e-68ed6f094547 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2 + reference: '' + d5e30d96-961c-4936-a5c3-3e802454f4bf: !Template + answer_choices: A ||| B + id: d5e30d96-961c-4936-a5c3-3e802454f4bf + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 1 (good-bad, answer: A)' + reference: '' + fb6a991a-cce6-4ed3-a067-06ff84488bbe: !Template + answer_choices: null + id: fb6a991a-cce6-4ed3-a067-06ff84488bbe + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B + reference: '' diff --git a/promptsource/templates/blimp/determiner_noun_agreement_irregular_1/templates.yaml b/promptsource/templates/blimp/determiner_noun_agreement_irregular_1/templates.yaml new file mode 100644 index 000000000..eadfef437 --- /dev/null +++ b/promptsource/templates/blimp/determiner_noun_agreement_irregular_1/templates.yaml @@ -0,0 +1,235 @@ +dataset: blimp +subset: determiner_noun_agreement_irregular_1 +templates: + 0f092653-5b45-4dc2-9e5f-7406906cb12d: !Template + answer_choices: A ||| B + id: 0f092653-5b45-4dc2-9e5f-7406906cb12d + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 2 (bad-good, answer: B)' + reference: '' + 2b12b43a-b56e-40bc-99b0-ea0c61197caa: !Template + answer_choices: Yes ||| No + id: 2b12b43a-b56e-40bc-99b0-ea0c61197caa + jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' + 64fad99b-1f26-47dc-a376-be3f54f24517: !Template + answer_choices: null + id: 64fad99b-1f26-47dc-a376-be3f54f24517 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2 + reference: '' + 780b7420-7712-4a28-a78e-edfcc051ebaf: !Template + answer_choices: B ||| A + id: 780b7420-7712-4a28-a78e-edfcc051ebaf + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice, 2 (bad-good, answer: B)' + reference: '' + 7bebfb37-eb3f-4e16-821d-8ab8e3e97a20: !Template + answer_choices: null + id: 7bebfb37-eb3f-4e16-821d-8ab8e3e97a20 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B + reference: '' + ca54c469-745a-4918-987f-84fd2bf2e1dd: !Template + answer_choices: null + id: ca54c469-745a-4918-987f-84fd2bf2e1dd + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2_2 + reference: '' + cb6a79a1-8f0b-4762-b8a7-219352fc85f3: !Template + answer_choices: null + id: cb6a79a1-8f0b-4762-b8a7-219352fc85f3 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B_2 + reference: '' + e1066a6d-41ef-4693-8ce9-6e5abe3ad19d: !Template + answer_choices: Yes ||| No + id: e1066a6d-41ef-4693-8ce9-6e5abe3ad19d + jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' + ed1ba6fd-9a83-45a6-9e5c-acbcc42c8543: !Template + answer_choices: B ||| A + id: ed1ba6fd-9a83-45a6-9e5c-acbcc42c8543 + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice 1 (good-bad, answer: A)' + reference: '' + fbd32631-90e3-418a-8a8c-430bae7cb8c3: !Template + answer_choices: A ||| B + id: fbd32631-90e3-418a-8a8c-430bae7cb8c3 + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 1 (good-bad, answer: A)' + reference: '' diff --git a/promptsource/templates/blimp/determiner_noun_agreement_irregular_2/templates.yaml b/promptsource/templates/blimp/determiner_noun_agreement_irregular_2/templates.yaml new file mode 100644 index 000000000..5db2b87a7 --- /dev/null +++ b/promptsource/templates/blimp/determiner_noun_agreement_irregular_2/templates.yaml @@ -0,0 +1,235 @@ +dataset: blimp +subset: determiner_noun_agreement_irregular_2 +templates: + 097e2730-3125-4fb0-965d-fefc2f12fc85: !Template + answer_choices: null + id: 097e2730-3125-4fb0-965d-fefc2f12fc85 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B_2 + reference: '' + 19865635-e759-4a45-ba10-ba837c168d1f: !Template + answer_choices: B ||| A + id: 19865635-e759-4a45-ba10-ba837c168d1f + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice, 2 (bad-good, answer: B)' + reference: '' + 3a1ce220-3539-41e3-8288-44dad0ac6322: !Template + answer_choices: B ||| A + id: 3a1ce220-3539-41e3-8288-44dad0ac6322 + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice 1 (good-bad, answer: A)' + reference: '' + 3f5c5974-3f16-4a1c-993a-970d6a39b5cf: !Template + answer_choices: null + id: 3f5c5974-3f16-4a1c-993a-970d6a39b5cf + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2 + reference: '' + 4beb8a18-9833-40a7-91a2-79b8c88afa20: !Template + answer_choices: null + id: 4beb8a18-9833-40a7-91a2-79b8c88afa20 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B + reference: '' + 59c0367d-ac9e-41e7-8a12-919de9aab7fc: !Template + answer_choices: null + id: 59c0367d-ac9e-41e7-8a12-919de9aab7fc + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2_2 + reference: '' + 7818b45f-0003-49c6-aa61-c50d1b6ae640: !Template + answer_choices: A ||| B + id: 7818b45f-0003-49c6-aa61-c50d1b6ae640 + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 1 (good-bad, answer: A)' + reference: '' + bc667eda-3467-45f5-baac-f2dc79e33834: !Template + answer_choices: Yes ||| No + id: bc667eda-3467-45f5-baac-f2dc79e33834 + jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' + d88ba508-a5b2-4ffd-a65b-ef7cf57b3066: !Template + answer_choices: Yes ||| No + id: d88ba508-a5b2-4ffd-a65b-ef7cf57b3066 + jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' + eac5b2f5-db99-48a4-9915-64342d4d0f74: !Template + answer_choices: A ||| B + id: eac5b2f5-db99-48a4-9915-64342d4d0f74 + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 2 (bad-good, answer: B)' + reference: '' diff --git a/promptsource/templates/blimp/determiner_noun_agreement_with_adj_2/templates.yaml b/promptsource/templates/blimp/determiner_noun_agreement_with_adj_2/templates.yaml new file mode 100644 index 000000000..fe68b7541 --- /dev/null +++ b/promptsource/templates/blimp/determiner_noun_agreement_with_adj_2/templates.yaml @@ -0,0 +1,235 @@ +dataset: blimp +subset: determiner_noun_agreement_with_adj_2 +templates: + 13ca673f-eb0e-4ae9-be69-4e41a991ff33: !Template + answer_choices: Yes ||| No + id: 13ca673f-eb0e-4ae9-be69-4e41a991ff33 + jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' + 4914bacd-a781-4b71-b73f-0db38535629b: !Template + answer_choices: null + id: 4914bacd-a781-4b71-b73f-0db38535629b + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B + reference: '' + 598ea465-01d3-48ce-9608-18f4829129ca: !Template + answer_choices: null + id: 598ea465-01d3-48ce-9608-18f4829129ca + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2 + reference: '' + 5b79d519-9e1d-4235-9a55-9dc67928bafb: !Template + answer_choices: null + id: 5b79d519-9e1d-4235-9a55-9dc67928bafb + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B_2 + reference: '' + 766aa0e2-f2e9-431b-9ff0-54592092696f: !Template + answer_choices: null + id: 766aa0e2-f2e9-431b-9ff0-54592092696f + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2_2 + reference: '' + bc33bb59-e1ef-4fc9-85b5-e47a2b1c62b8: !Template + answer_choices: B ||| A + id: bc33bb59-e1ef-4fc9-85b5-e47a2b1c62b8 + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice 1 (good-bad, answer: A)' + reference: '' + cfb5185c-6af7-4595-9edb-cee461744fd9: !Template + answer_choices: A ||| B + id: cfb5185c-6af7-4595-9edb-cee461744fd9 + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 1 (good-bad, answer: A)' + reference: '' + e1c6e2f4-d204-4c66-899f-a1859c8bc95e: !Template + answer_choices: Yes ||| No + id: e1c6e2f4-d204-4c66-899f-a1859c8bc95e + jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' + f9af577a-73f4-4f7e-94b6-33821626c423: !Template + answer_choices: B ||| A + id: f9af577a-73f4-4f7e-94b6-33821626c423 + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice, 2 (bad-good, answer: B)' + reference: '' + fc68020f-df74-4afd-87e4-b7a92a79b87b: !Template + answer_choices: A ||| B + id: fc68020f-df74-4afd-87e4-b7a92a79b87b + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 2 (bad-good, answer: B)' + reference: '' diff --git a/promptsource/templates/blimp/determiner_noun_agreement_with_adj_irregular_1/templates.yaml b/promptsource/templates/blimp/determiner_noun_agreement_with_adj_irregular_1/templates.yaml new file mode 100644 index 000000000..c78b330e4 --- /dev/null +++ b/promptsource/templates/blimp/determiner_noun_agreement_with_adj_irregular_1/templates.yaml @@ -0,0 +1,235 @@ +dataset: blimp +subset: determiner_noun_agreement_with_adj_irregular_1 +templates: + 150bd9cf-b9c2-4417-aace-90be474c5ff3: !Template + answer_choices: B ||| A + id: 150bd9cf-b9c2-4417-aace-90be474c5ff3 + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice, 2 (bad-good, answer: B)' + reference: '' + 18403cac-8fb3-4c8c-b455-e0f79f0f5e6b: !Template + answer_choices: A ||| B + id: 18403cac-8fb3-4c8c-b455-e0f79f0f5e6b + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 1 (good-bad, answer: A)' + reference: '' + 286df295-a069-4d1e-b29d-417b9d39b044: !Template + answer_choices: Yes ||| No + id: 286df295-a069-4d1e-b29d-417b9d39b044 + jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' + 51581cf6-6f89-4f61-aaed-7886fc2f127c: !Template + answer_choices: null + id: 51581cf6-6f89-4f61-aaed-7886fc2f127c + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B_2 + reference: '' + 597ec04b-0f83-4401-81e4-00eb67c5f876: !Template + answer_choices: null + id: 597ec04b-0f83-4401-81e4-00eb67c5f876 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B + reference: '' + 62c66714-fd11-4070-868e-56c5c5089d62: !Template + answer_choices: null + id: 62c66714-fd11-4070-868e-56c5c5089d62 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2_2 + reference: '' + 913ace8e-782a-43a9-b9b3-b5117170d119: !Template + answer_choices: Yes ||| No + id: 913ace8e-782a-43a9-b9b3-b5117170d119 + jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' + 9f0628cc-504c-4c70-9540-0713774e3d3d: !Template + answer_choices: null + id: 9f0628cc-504c-4c70-9540-0713774e3d3d + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2 + reference: '' + a0890919-8ee2-4d28-896f-43abb4f0adc8: !Template + answer_choices: A ||| B + id: a0890919-8ee2-4d28-896f-43abb4f0adc8 + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 2 (bad-good, answer: B)' + reference: '' + a676df72-b67a-4c3f-92a3-91a7d77da87c: !Template + answer_choices: B ||| A + id: a676df72-b67a-4c3f-92a3-91a7d77da87c + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice 1 (good-bad, answer: A)' + reference: '' diff --git a/promptsource/templates/blimp/determiner_noun_agreement_with_adj_irregular_2/templates.yaml b/promptsource/templates/blimp/determiner_noun_agreement_with_adj_irregular_2/templates.yaml new file mode 100644 index 000000000..c7138ee3e --- /dev/null +++ b/promptsource/templates/blimp/determiner_noun_agreement_with_adj_irregular_2/templates.yaml @@ -0,0 +1,235 @@ +dataset: blimp +subset: determiner_noun_agreement_with_adj_irregular_2 +templates: + 232ce8bd-94ad-498d-97dd-262b3ec93353: !Template + answer_choices: A ||| B + id: 232ce8bd-94ad-498d-97dd-262b3ec93353 + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 2 (bad-good, answer: B)' + reference: '' + 32c7010c-bb30-4279-9b29-4badb633afce: !Template + answer_choices: null + id: 32c7010c-bb30-4279-9b29-4badb633afce + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2 + reference: '' + 3433b964-9c02-476d-97ca-90398e49172e: !Template + answer_choices: null + id: 3433b964-9c02-476d-97ca-90398e49172e + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2_2 + reference: '' + 54dd2295-1f1b-4bb0-90bc-e5b14478267f: !Template + answer_choices: Yes ||| No + id: 54dd2295-1f1b-4bb0-90bc-e5b14478267f + jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' + 9eeb1515-b05e-41ba-baf6-b861f1c0adcc: !Template + answer_choices: null + id: 9eeb1515-b05e-41ba-baf6-b861f1c0adcc + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B_2 + reference: '' + ac89930d-5eaa-497d-bc4f-1c2860c4e89e: !Template + answer_choices: B ||| A + id: ac89930d-5eaa-497d-bc4f-1c2860c4e89e + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice 1 (good-bad, answer: A)' + reference: '' + acda4d45-d598-467f-ae6f-b88a720e80a8: !Template + answer_choices: A ||| B + id: acda4d45-d598-467f-ae6f-b88a720e80a8 + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 1 (good-bad, answer: A)' + reference: '' + c8bfd8f3-f57c-406d-a60d-d2a7aaa8597e: !Template + answer_choices: B ||| A + id: c8bfd8f3-f57c-406d-a60d-d2a7aaa8597e + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice, 2 (bad-good, answer: B)' + reference: '' + f5e48fb8-5441-48c5-814f-5622bac948f8: !Template + answer_choices: Yes ||| No + id: f5e48fb8-5441-48c5-814f-5622bac948f8 + jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' + f7adc7ba-afcc-48ab-9d4f-d5bb93225515: !Template + answer_choices: null + id: f7adc7ba-afcc-48ab-9d4f-d5bb93225515 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B + reference: '' diff --git a/promptsource/templates/blimp/determiner_noun_agreement_with_adjective_1/templates.yaml b/promptsource/templates/blimp/determiner_noun_agreement_with_adjective_1/templates.yaml new file mode 100644 index 000000000..06beb3a74 --- /dev/null +++ b/promptsource/templates/blimp/determiner_noun_agreement_with_adjective_1/templates.yaml @@ -0,0 +1,235 @@ +dataset: blimp +subset: determiner_noun_agreement_with_adjective_1 +templates: + 1d4cb84a-790a-43f1-b145-bafe19d90677: !Template + answer_choices: null + id: 1d4cb84a-790a-43f1-b145-bafe19d90677 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2_2 + reference: '' + 1f60c65a-fb5e-488b-ac3b-9126138bc49b: !Template + answer_choices: null + id: 1f60c65a-fb5e-488b-ac3b-9126138bc49b + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2 + reference: '' + 51ba2249-12a3-40b0-a162-bbc6642c61fd: !Template + answer_choices: null + id: 51ba2249-12a3-40b0-a162-bbc6642c61fd + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B_2 + reference: '' + 6a190fe0-e353-43ac-922e-02b95cc8e8c3: !Template + answer_choices: A ||| B + id: 6a190fe0-e353-43ac-922e-02b95cc8e8c3 + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 2 (bad-good, answer: B)' + reference: '' + 7a11abee-2d43-40ba-9d77-0714ddea41e3: !Template + answer_choices: null + id: 7a11abee-2d43-40ba-9d77-0714ddea41e3 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B + reference: '' + 7c92f1b1-979e-41c5-918a-c24a0427c5a6: !Template + answer_choices: A ||| B + id: 7c92f1b1-979e-41c5-918a-c24a0427c5a6 + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 1 (good-bad, answer: A)' + reference: '' + 8f8db402-b5e7-44d3-942a-7c6f260d0356: !Template + answer_choices: Yes ||| No + id: 8f8db402-b5e7-44d3-942a-7c6f260d0356 + jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' + cd6f0265-e95b-4aa3-9596-fb17f8a5205e: !Template + answer_choices: Yes ||| No + id: cd6f0265-e95b-4aa3-9596-fb17f8a5205e + jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' + f95ccc81-68ee-45de-9f20-30b1a8edfe02: !Template + answer_choices: B ||| A + id: f95ccc81-68ee-45de-9f20-30b1a8edfe02 + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice 1 (good-bad, answer: A)' + reference: '' + fe6e92d5-7f2d-4155-a90f-9800adb3deb9: !Template + answer_choices: B ||| A + id: fe6e92d5-7f2d-4155-a90f-9800adb3deb9 + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice, 2 (bad-good, answer: B)' + reference: '' diff --git a/promptsource/templates/blimp/distractor_agreement_relational_noun/templates.yaml b/promptsource/templates/blimp/distractor_agreement_relational_noun/templates.yaml new file mode 100644 index 000000000..5ca416469 --- /dev/null +++ b/promptsource/templates/blimp/distractor_agreement_relational_noun/templates.yaml @@ -0,0 +1,235 @@ +dataset: blimp +subset: distractor_agreement_relational_noun +templates: + 07170e39-a4cd-4fec-95a0-29c37e8d6504: !Template + answer_choices: B ||| A + id: 07170e39-a4cd-4fec-95a0-29c37e8d6504 + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice 1 (good-bad, answer: A)' + reference: '' + 33f9426d-25f5-420f-ba89-8b420020861d: !Template + answer_choices: null + id: 33f9426d-25f5-420f-ba89-8b420020861d + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B + reference: '' + 39b4ee35-30db-4618-bbc8-0327a12044e4: !Template + answer_choices: A ||| B + id: 39b4ee35-30db-4618-bbc8-0327a12044e4 + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 2 (bad-good, answer: B)' + reference: '' + 3e9d7a9e-e369-4c15-a378-18b6b2caa9b4: !Template + answer_choices: null + id: 3e9d7a9e-e369-4c15-a378-18b6b2caa9b4 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2_2 + reference: '' + 96c27465-15fb-4542-ad24-960d0f7f6dc8: !Template + answer_choices: Yes ||| No + id: 96c27465-15fb-4542-ad24-960d0f7f6dc8 + jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' + b8e3fb6f-8dd4-4898-828a-3a0f8cb5f26c: !Template + answer_choices: B ||| A + id: b8e3fb6f-8dd4-4898-828a-3a0f8cb5f26c + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice, 2 (bad-good, answer: B)' + reference: '' + b9f81964-631c-4380-923a-2505ca8e1893: !Template + answer_choices: null + id: b9f81964-631c-4380-923a-2505ca8e1893 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2 + reference: '' + c8b8d721-878b-4e3d-acbc-ced91ccb2de5: !Template + answer_choices: null + id: c8b8d721-878b-4e3d-acbc-ced91ccb2de5 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B_2 + reference: '' + ea246dd2-a17a-49ba-8deb-022e460c042d: !Template + answer_choices: A ||| B + id: ea246dd2-a17a-49ba-8deb-022e460c042d + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 1 (good-bad, answer: A)' + reference: '' + fbbd640c-dc26-44ea-8a22-312d182cd032: !Template + answer_choices: Yes ||| No + id: fbbd640c-dc26-44ea-8a22-312d182cd032 + jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' diff --git a/promptsource/templates/blimp/distractor_agreement_relative_clause/templates.yaml b/promptsource/templates/blimp/distractor_agreement_relative_clause/templates.yaml new file mode 100644 index 000000000..2cd354c14 --- /dev/null +++ b/promptsource/templates/blimp/distractor_agreement_relative_clause/templates.yaml @@ -0,0 +1,235 @@ +dataset: blimp +subset: distractor_agreement_relative_clause +templates: + 1d3638c5-3b0b-42e0-9c1d-47f39f78deae: !Template + answer_choices: null + id: 1d3638c5-3b0b-42e0-9c1d-47f39f78deae + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B_2 + reference: '' + 4261ba02-db77-4161-b9ff-272178b64a02: !Template + answer_choices: A ||| B + id: 4261ba02-db77-4161-b9ff-272178b64a02 + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 2 (bad-good, answer: B)' + reference: '' + 4409efc6-3653-4b75-8e1a-518c4443b94b: !Template + answer_choices: Yes ||| No + id: 4409efc6-3653-4b75-8e1a-518c4443b94b + jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' + 455e2c40-1fdc-46bf-a754-a0d066f50566: !Template + answer_choices: Yes ||| No + id: 455e2c40-1fdc-46bf-a754-a0d066f50566 + jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' + 7b0308ea-1e07-49ec-9aac-e28ab4ef96da: !Template + answer_choices: null + id: 7b0308ea-1e07-49ec-9aac-e28ab4ef96da + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2 + reference: '' + 90f02563-e7f5-4ebf-9476-6b7d84b240df: !Template + answer_choices: B ||| A + id: 90f02563-e7f5-4ebf-9476-6b7d84b240df + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice, 2 (bad-good, answer: B)' + reference: '' + 928cc6a0-9fa0-4fa9-baf7-b7d2ee2c453b: !Template + answer_choices: B ||| A + id: 928cc6a0-9fa0-4fa9-baf7-b7d2ee2c453b + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice 1 (good-bad, answer: A)' + reference: '' + a3c584cb-35a6-417e-92c7-25e5015f2e21: !Template + answer_choices: null + id: a3c584cb-35a6-417e-92c7-25e5015f2e21 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2_2 + reference: '' + a66605d0-f318-4a5f-bba7-d2df67f01d6d: !Template + answer_choices: A ||| B + id: a66605d0-f318-4a5f-bba7-d2df67f01d6d + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 1 (good-bad, answer: A)' + reference: '' + cf595472-4ecf-4147-b76d-cb47fc879884: !Template + answer_choices: null + id: cf595472-4ecf-4147-b76d-cb47fc879884 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B + reference: '' diff --git a/promptsource/templates/blimp/drop_argument/templates.yaml b/promptsource/templates/blimp/drop_argument/templates.yaml new file mode 100644 index 000000000..66035c31b --- /dev/null +++ b/promptsource/templates/blimp/drop_argument/templates.yaml @@ -0,0 +1,235 @@ +dataset: blimp +subset: drop_argument +templates: + 347ccb81-2e47-4cae-8d0a-736888d89ff8: !Template + answer_choices: A ||| B + id: 347ccb81-2e47-4cae-8d0a-736888d89ff8 + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 1 (good-bad, answer: A)' + reference: '' + 3f3c1792-3a68-475d-a817-5219773dae8d: !Template + answer_choices: B ||| A + id: 3f3c1792-3a68-475d-a817-5219773dae8d + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice 1 (good-bad, answer: A)' + reference: '' + 6501fec5-fb69-414a-9eee-75587833f1a0: !Template + answer_choices: Yes ||| No + id: 6501fec5-fb69-414a-9eee-75587833f1a0 + jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' + 72e37f51-2c07-43b6-a5bc-305e89b58d9c: !Template + answer_choices: Yes ||| No + id: 72e37f51-2c07-43b6-a5bc-305e89b58d9c + jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' + 75f965d1-8cc7-49d0-b496-7eee15116642: !Template + answer_choices: null + id: 75f965d1-8cc7-49d0-b496-7eee15116642 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B + reference: '' + a5553fc9-c58a-4a8f-adb4-35ed4a1510a0: !Template + answer_choices: null + id: a5553fc9-c58a-4a8f-adb4-35ed4a1510a0 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2_2 + reference: '' + b7709a79-a79e-42d6-b34c-816d5ee4221c: !Template + answer_choices: null + id: b7709a79-a79e-42d6-b34c-816d5ee4221c + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B_2 + reference: '' + cffaca3b-3201-413c-9eaf-23a19714ef69: !Template + answer_choices: A ||| B + id: cffaca3b-3201-413c-9eaf-23a19714ef69 + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 2 (bad-good, answer: B)' + reference: '' + e859c136-33c7-426e-bb67-a48481d452ce: !Template + answer_choices: null + id: e859c136-33c7-426e-bb67-a48481d452ce + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2 + reference: '' + fc90633d-b1ad-43e8-9874-223f5d59ac0a: !Template + answer_choices: B ||| A + id: fc90633d-b1ad-43e8-9874-223f5d59ac0a + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice, 2 (bad-good, answer: B)' + reference: '' diff --git a/promptsource/templates/blimp/ellipsis_n_bar_1/templates.yaml b/promptsource/templates/blimp/ellipsis_n_bar_1/templates.yaml new file mode 100644 index 000000000..7525b87bd --- /dev/null +++ b/promptsource/templates/blimp/ellipsis_n_bar_1/templates.yaml @@ -0,0 +1,235 @@ +dataset: blimp +subset: ellipsis_n_bar_1 +templates: + 248f9e3f-9bdb-456b-add3-31b9b18db5fa: !Template + answer_choices: Yes ||| No + id: 248f9e3f-9bdb-456b-add3-31b9b18db5fa + jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' + 327a1cc7-80d0-4a99-a0c9-0f7207beb81c: !Template + answer_choices: null + id: 327a1cc7-80d0-4a99-a0c9-0f7207beb81c + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B_2 + reference: '' + 8b6db999-81c8-4c88-ae9a-de07cd958b6f: !Template + answer_choices: B ||| A + id: 8b6db999-81c8-4c88-ae9a-de07cd958b6f + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice 1 (good-bad, answer: A)' + reference: '' + 9a7f3418-0e1b-4981-8059-83981dcc4ab4: !Template + answer_choices: null + id: 9a7f3418-0e1b-4981-8059-83981dcc4ab4 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2 + reference: '' + cce1cedf-4fa6-4a7d-8b07-07e845ee8d18: !Template + answer_choices: A ||| B + id: cce1cedf-4fa6-4a7d-8b07-07e845ee8d18 + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 1 (good-bad, answer: A)' + reference: '' + e0e0b64f-851b-495a-9716-91e950479bc1: !Template + answer_choices: Yes ||| No + id: e0e0b64f-851b-495a-9716-91e950479bc1 + jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' + e9b63520-d889-474a-92f8-fdde852b1af6: !Template + answer_choices: null + id: e9b63520-d889-474a-92f8-fdde852b1af6 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B + reference: '' + f80d1616-daec-4448-b30c-9ae3251965ad: !Template + answer_choices: B ||| A + id: f80d1616-daec-4448-b30c-9ae3251965ad + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice, 2 (bad-good, answer: B)' + reference: '' + ff8f539f-13f6-45a6-ba3b-6ce5ecdf1947: !Template + answer_choices: A ||| B + id: ff8f539f-13f6-45a6-ba3b-6ce5ecdf1947 + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 2 (bad-good, answer: B)' + reference: '' + ffa392d4-c1b6-45d7-95d8-5a0f67fbafcd: !Template + answer_choices: null + id: ffa392d4-c1b6-45d7-95d8-5a0f67fbafcd + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2_2 + reference: '' diff --git a/promptsource/templates/blimp/ellipsis_n_bar_2/templates.yaml b/promptsource/templates/blimp/ellipsis_n_bar_2/templates.yaml new file mode 100644 index 000000000..b790320c4 --- /dev/null +++ b/promptsource/templates/blimp/ellipsis_n_bar_2/templates.yaml @@ -0,0 +1,235 @@ +dataset: blimp +subset: ellipsis_n_bar_2 +templates: + 042361b3-06b8-4ae9-a3e8-1f14b7be1306: !Template + answer_choices: A ||| B + id: 042361b3-06b8-4ae9-a3e8-1f14b7be1306 + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 2 (bad-good, answer: B)' + reference: '' + 06940021-1f86-42f2-bfeb-f4c019a21578: !Template + answer_choices: null + id: 06940021-1f86-42f2-bfeb-f4c019a21578 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B_2 + reference: '' + 13411dd1-e05f-464c-b8f1-40dd3d533ac6: !Template + answer_choices: B ||| A + id: 13411dd1-e05f-464c-b8f1-40dd3d533ac6 + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice, 2 (bad-good, answer: B)' + reference: '' + 25102e92-f738-455b-af3d-ee798b1ddfcd: !Template + answer_choices: Yes ||| No + id: 25102e92-f738-455b-af3d-ee798b1ddfcd + jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' + 31891c60-4c57-4c5d-9146-77c8795c2d84: !Template + answer_choices: B ||| A + id: 31891c60-4c57-4c5d-9146-77c8795c2d84 + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice 1 (good-bad, answer: A)' + reference: '' + 71113537-7e9b-4f83-a5b2-d1f6ea8b07e8: !Template + answer_choices: null + id: 71113537-7e9b-4f83-a5b2-d1f6ea8b07e8 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2_2 + reference: '' + 812791bf-dad4-481c-8dca-1fff8626799b: !Template + answer_choices: null + id: 812791bf-dad4-481c-8dca-1fff8626799b + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B + reference: '' + 990f63ec-4e22-40e9-8178-e58454dc3ede: !Template + answer_choices: Yes ||| No + id: 990f63ec-4e22-40e9-8178-e58454dc3ede + jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' + a11f17a4-6e2d-4b5d-9833-b73dcd987702: !Template + answer_choices: null + id: a11f17a4-6e2d-4b5d-9833-b73dcd987702 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2 + reference: '' + b87e7cc8-ce0a-4dee-a256-05d61fa2efbb: !Template + answer_choices: A ||| B + id: b87e7cc8-ce0a-4dee-a256-05d61fa2efbb + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 1 (good-bad, answer: A)' + reference: '' diff --git a/promptsource/templates/blimp/existential_there_object_raising/templates.yaml b/promptsource/templates/blimp/existential_there_object_raising/templates.yaml new file mode 100644 index 000000000..e4dd329ae --- /dev/null +++ b/promptsource/templates/blimp/existential_there_object_raising/templates.yaml @@ -0,0 +1,235 @@ +dataset: blimp +subset: existential_there_object_raising +templates: + 01e4ede6-3cf1-443b-ad82-69b7f67112ac: !Template + answer_choices: Yes ||| No + id: 01e4ede6-3cf1-443b-ad82-69b7f67112ac + jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' + 278e0258-a74e-43c0-8b6e-c1cda8c298a7: !Template + answer_choices: Yes ||| No + id: 278e0258-a74e-43c0-8b6e-c1cda8c298a7 + jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' + 4c318fc5-fd44-4fda-9dac-24eb65793fcf: !Template + answer_choices: null + id: 4c318fc5-fd44-4fda-9dac-24eb65793fcf + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B + reference: '' + 4eb61e9c-417f-48f4-aae8-bb7f82e02f8d: !Template + answer_choices: null + id: 4eb61e9c-417f-48f4-aae8-bb7f82e02f8d + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B_2 + reference: '' + 7b0dc8bf-eba0-4c7a-b7f0-29b276f36ae6: !Template + answer_choices: A ||| B + id: 7b0dc8bf-eba0-4c7a-b7f0-29b276f36ae6 + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 1 (good-bad, answer: A)' + reference: '' + 7b2e11e9-0417-4c96-8b67-dde9a884ca5d: !Template + answer_choices: null + id: 7b2e11e9-0417-4c96-8b67-dde9a884ca5d + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2_2 + reference: '' + 9dd2b833-7a83-4024-8371-b4abcc3744b1: !Template + answer_choices: B ||| A + id: 9dd2b833-7a83-4024-8371-b4abcc3744b1 + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice, 2 (bad-good, answer: B)' + reference: '' + 9f1529b8-a026-46b4-88a4-0dc78b2c1873: !Template + answer_choices: A ||| B + id: 9f1529b8-a026-46b4-88a4-0dc78b2c1873 + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 2 (bad-good, answer: B)' + reference: '' + a6fca9f8-42cb-4ff4-b264-bc6171793913: !Template + answer_choices: B ||| A + id: a6fca9f8-42cb-4ff4-b264-bc6171793913 + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice 1 (good-bad, answer: A)' + reference: '' + d43034bb-0349-462e-8b96-077297c7221e: !Template + answer_choices: null + id: d43034bb-0349-462e-8b96-077297c7221e + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2 + reference: '' diff --git a/promptsource/templates/blimp/existential_there_quantifiers_1/templates.yaml b/promptsource/templates/blimp/existential_there_quantifiers_1/templates.yaml new file mode 100644 index 000000000..04e7b9f4c --- /dev/null +++ b/promptsource/templates/blimp/existential_there_quantifiers_1/templates.yaml @@ -0,0 +1,235 @@ +dataset: blimp +subset: existential_there_quantifiers_1 +templates: + 00750be4-7bbf-4358-b7e1-d601a6f3a545: !Template + answer_choices: Yes ||| No + id: 00750be4-7bbf-4358-b7e1-d601a6f3a545 + jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' + 6c44cd14-a0cb-47ec-b274-5b69db0f8291: !Template + answer_choices: null + id: 6c44cd14-a0cb-47ec-b274-5b69db0f8291 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B_2 + reference: '' + 6e7cf462-c2ff-49c6-b986-0ba9003be9cc: !Template + answer_choices: B ||| A + id: 6e7cf462-c2ff-49c6-b986-0ba9003be9cc + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice, 2 (bad-good, answer: B)' + reference: '' + 80abf7a6-8b8c-4a15-956c-5bb3aee4b84e: !Template + answer_choices: null + id: 80abf7a6-8b8c-4a15-956c-5bb3aee4b84e + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2 + reference: '' + 9fbb74c6-7d48-40f7-9262-ec05ca9ea1e0: !Template + answer_choices: null + id: 9fbb74c6-7d48-40f7-9262-ec05ca9ea1e0 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2_2 + reference: '' + b4e82d40-7e52-4118-ad35-c2022b8faec7: !Template + answer_choices: B ||| A + id: b4e82d40-7e52-4118-ad35-c2022b8faec7 + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice 1 (good-bad, answer: A)' + reference: '' + bede45a1-5834-4444-8c52-986f3c273a04: !Template + answer_choices: Yes ||| No + id: bede45a1-5834-4444-8c52-986f3c273a04 + jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' + e1f0143e-ac66-4187-bf06-50835d581592: !Template + answer_choices: A ||| B + id: e1f0143e-ac66-4187-bf06-50835d581592 + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 2 (bad-good, answer: B)' + reference: '' + f0785228-7647-450a-b6b8-11e7f9120bbe: !Template + answer_choices: A ||| B + id: f0785228-7647-450a-b6b8-11e7f9120bbe + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 1 (good-bad, answer: A)' + reference: '' + f373c4a6-1782-4a00-b513-c960f2d57dfc: !Template + answer_choices: null + id: f373c4a6-1782-4a00-b513-c960f2d57dfc + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B + reference: '' diff --git a/promptsource/templates/blimp/existential_there_quantifiers_2/templates.yaml b/promptsource/templates/blimp/existential_there_quantifiers_2/templates.yaml new file mode 100644 index 000000000..d34d97e23 --- /dev/null +++ b/promptsource/templates/blimp/existential_there_quantifiers_2/templates.yaml @@ -0,0 +1,235 @@ +dataset: blimp +subset: existential_there_quantifiers_2 +templates: + 105555b0-4d08-4590-94e7-12ad3343ea1e: !Template + answer_choices: A ||| B + id: 105555b0-4d08-4590-94e7-12ad3343ea1e + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 1 (good-bad, answer: A)' + reference: '' + 1fcb97d3-5ed3-4c61-9421-43354409949e: !Template + answer_choices: B ||| A + id: 1fcb97d3-5ed3-4c61-9421-43354409949e + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice, 2 (bad-good, answer: B)' + reference: '' + 5ca43b41-0672-4b19-8808-26fe4e21b307: !Template + answer_choices: null + id: 5ca43b41-0672-4b19-8808-26fe4e21b307 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2_2 + reference: '' + 69ebf4bc-ca4a-4e0b-b448-4b90baba3d72: !Template + answer_choices: null + id: 69ebf4bc-ca4a-4e0b-b448-4b90baba3d72 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B_2 + reference: '' + d2f78114-9249-4777-bfef-fb3c457ea14b: !Template + answer_choices: null + id: d2f78114-9249-4777-bfef-fb3c457ea14b + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2 + reference: '' + dd960fc2-ff85-4d96-893c-e9cc37d95b61: !Template + answer_choices: Yes ||| No + id: dd960fc2-ff85-4d96-893c-e9cc37d95b61 + jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' + f033067b-e2bc-4e53-b5f4-e614767a6c13: !Template + answer_choices: null + id: f033067b-e2bc-4e53-b5f4-e614767a6c13 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B + reference: '' + f1405ef4-a250-4981-9d68-3dc2d2d0e583: !Template + answer_choices: Yes ||| No + id: f1405ef4-a250-4981-9d68-3dc2d2d0e583 + jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' + fc212969-d0cf-4942-aaea-75b55d3f453f: !Template + answer_choices: B ||| A + id: fc212969-d0cf-4942-aaea-75b55d3f453f + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice 1 (good-bad, answer: A)' + reference: '' + fe547b2b-12f2-4fef-a3f6-c2e45dce6309: !Template + answer_choices: A ||| B + id: fe547b2b-12f2-4fef-a3f6-c2e45dce6309 + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 2 (bad-good, answer: B)' + reference: '' diff --git a/promptsource/templates/blimp/existential_there_subject_raising/templates.yaml b/promptsource/templates/blimp/existential_there_subject_raising/templates.yaml new file mode 100644 index 000000000..c845a9cbc --- /dev/null +++ b/promptsource/templates/blimp/existential_there_subject_raising/templates.yaml @@ -0,0 +1,235 @@ +dataset: blimp +subset: existential_there_subject_raising +templates: + 003be84d-8ca5-4e8f-8507-6eedee51025c: !Template + answer_choices: B ||| A + id: 003be84d-8ca5-4e8f-8507-6eedee51025c + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice, 2 (bad-good, answer: B)' + reference: '' + 3e7e588f-c8a9-4eff-aa3b-9cec2dd1b82c: !Template + answer_choices: Yes ||| No + id: 3e7e588f-c8a9-4eff-aa3b-9cec2dd1b82c + jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' + 41382acf-6f8d-4835-aee7-7dbe0d5d1c7b: !Template + answer_choices: A ||| B + id: 41382acf-6f8d-4835-aee7-7dbe0d5d1c7b + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 1 (good-bad, answer: A)' + reference: '' + 41ae3a18-ed4e-4cd1-8dc5-ef844b5f5c1d: !Template + answer_choices: null + id: 41ae3a18-ed4e-4cd1-8dc5-ef844b5f5c1d + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B_2 + reference: '' + 538bbe9b-e66c-4fb9-b69f-d15b9925e671: !Template + answer_choices: B ||| A + id: 538bbe9b-e66c-4fb9-b69f-d15b9925e671 + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice 1 (good-bad, answer: A)' + reference: '' + 68a68f4e-6347-4dfa-a8fb-06e9a3a42eaf: !Template + answer_choices: A ||| B + id: 68a68f4e-6347-4dfa-a8fb-06e9a3a42eaf + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 2 (bad-good, answer: B)' + reference: '' + 6e252204-c288-4377-94cc-eba6c2238ce6: !Template + answer_choices: null + id: 6e252204-c288-4377-94cc-eba6c2238ce6 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2_2 + reference: '' + 7295849e-7a4c-45e4-a551-64ee347a51c3: !Template + answer_choices: null + id: 7295849e-7a4c-45e4-a551-64ee347a51c3 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2 + reference: '' + 7977d568-25a4-4fb6-9486-02e851bbbeae: !Template + answer_choices: null + id: 7977d568-25a4-4fb6-9486-02e851bbbeae + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B + reference: '' + 8823b041-ea11-45a6-b1ff-f3ca2df932f9: !Template + answer_choices: Yes ||| No + id: 8823b041-ea11-45a6-b1ff-f3ca2df932f9 + jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' diff --git a/promptsource/templates/blimp/expletive_it_object_raising/templates.yaml b/promptsource/templates/blimp/expletive_it_object_raising/templates.yaml new file mode 100644 index 000000000..0bf305cc3 --- /dev/null +++ b/promptsource/templates/blimp/expletive_it_object_raising/templates.yaml @@ -0,0 +1,235 @@ +dataset: blimp +subset: expletive_it_object_raising +templates: + 2488fd05-627b-4a63-947d-3853910430b3: !Template + answer_choices: B ||| A + id: 2488fd05-627b-4a63-947d-3853910430b3 + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice 1 (good-bad, answer: A)' + reference: '' + 296eb36b-7f17-4def-9c55-b23aa400dfee: !Template + answer_choices: Yes ||| No + id: 296eb36b-7f17-4def-9c55-b23aa400dfee + jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' + 5341302f-5e5c-41ac-a7cb-8843de9ad7a7: !Template + answer_choices: A ||| B + id: 5341302f-5e5c-41ac-a7cb-8843de9ad7a7 + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 1 (good-bad, answer: A)' + reference: '' + 709e0c6b-4e7f-44c7-97c4-cdcfe462ae04: !Template + answer_choices: null + id: 709e0c6b-4e7f-44c7-97c4-cdcfe462ae04 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2_2 + reference: '' + 9a386aab-430d-41b2-ab9e-af930cf8b627: !Template + answer_choices: A ||| B + id: 9a386aab-430d-41b2-ab9e-af930cf8b627 + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 2 (bad-good, answer: B)' + reference: '' + a2efd68d-dc4f-4ea6-bb2d-6717fab2dd71: !Template + answer_choices: B ||| A + id: a2efd68d-dc4f-4ea6-bb2d-6717fab2dd71 + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice, 2 (bad-good, answer: B)' + reference: '' + bb2ef94c-0607-4a0e-b4b7-97d98c3b0991: !Template + answer_choices: Yes ||| No + id: bb2ef94c-0607-4a0e-b4b7-97d98c3b0991 + jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' + e8d222f9-2f64-4d17-9a9d-92f47af11868: !Template + answer_choices: null + id: e8d222f9-2f64-4d17-9a9d-92f47af11868 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B_2 + reference: '' + f1e366ff-01e5-423b-8f6e-e5ec60a0981d: !Template + answer_choices: null + id: f1e366ff-01e5-423b-8f6e-e5ec60a0981d + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B + reference: '' + fcd45d25-2864-4031-bb0a-941e64c4f446: !Template + answer_choices: null + id: fcd45d25-2864-4031-bb0a-941e64c4f446 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2 + reference: '' diff --git a/promptsource/templates/blimp/inchoative/templates.yaml b/promptsource/templates/blimp/inchoative/templates.yaml new file mode 100644 index 000000000..26634c671 --- /dev/null +++ b/promptsource/templates/blimp/inchoative/templates.yaml @@ -0,0 +1,235 @@ +dataset: blimp +subset: inchoative +templates: + 5a6e8606-b48d-430d-8090-caec99767926: !Template + answer_choices: A ||| B + id: 5a6e8606-b48d-430d-8090-caec99767926 + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 1 (good-bad, answer: A)' + reference: '' + 5b9c18e3-2b8c-4826-b59c-736f5cf6cfdc: !Template + answer_choices: A ||| B + id: 5b9c18e3-2b8c-4826-b59c-736f5cf6cfdc + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 2 (bad-good, answer: B)' + reference: '' + 5e219d59-5066-4947-8368-070270606712: !Template + answer_choices: null + id: 5e219d59-5066-4947-8368-070270606712 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2 + reference: '' + 61fbe57c-44d7-4ff2-a2e5-5946d7df4c03: !Template + answer_choices: null + id: 61fbe57c-44d7-4ff2-a2e5-5946d7df4c03 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B_2 + reference: '' + 6fe5ff96-2d95-4d38-8ef6-9867b9b0f91f: !Template + answer_choices: Yes ||| No + id: 6fe5ff96-2d95-4d38-8ef6-9867b9b0f91f + jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' + 8818d4d4-9a03-4fcd-ba48-434f4af6fee7: !Template + answer_choices: null + id: 8818d4d4-9a03-4fcd-ba48-434f4af6fee7 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B + reference: '' + 9f67e427-081e-43ea-9ec2-66d48f0d7b4c: !Template + answer_choices: B ||| A + id: 9f67e427-081e-43ea-9ec2-66d48f0d7b4c + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice 1 (good-bad, answer: A)' + reference: '' + aaabaf9c-600d-44f0-95f6-4865dc120c8c: !Template + answer_choices: null + id: aaabaf9c-600d-44f0-95f6-4865dc120c8c + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2_2 + reference: '' + e0256a68-5e99-4709-bcfd-88aa6f057e11: !Template + answer_choices: Yes ||| No + id: e0256a68-5e99-4709-bcfd-88aa6f057e11 + jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' + e7d4ffcd-d5ea-4ffa-8d48-900c0156d756: !Template + answer_choices: B ||| A + id: e7d4ffcd-d5ea-4ffa-8d48-900c0156d756 + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice, 2 (bad-good, answer: B)' + reference: '' diff --git a/promptsource/templates/blimp/intransitive/templates.yaml b/promptsource/templates/blimp/intransitive/templates.yaml new file mode 100644 index 000000000..84cb8d38a --- /dev/null +++ b/promptsource/templates/blimp/intransitive/templates.yaml @@ -0,0 +1,235 @@ +dataset: blimp +subset: intransitive +templates: + 18c22644-daa2-4f28-bd44-08e6a8a6baac: !Template + answer_choices: Yes ||| No + id: 18c22644-daa2-4f28-bd44-08e6a8a6baac + jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' + 445d5527-da06-41ea-9cf2-0bdbe8b0b99c: !Template + answer_choices: A ||| B + id: 445d5527-da06-41ea-9cf2-0bdbe8b0b99c + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 1 (good-bad, answer: A)' + reference: '' + 64bc634f-909c-43a7-9028-55b4950e4d28: !Template + answer_choices: B ||| A + id: 64bc634f-909c-43a7-9028-55b4950e4d28 + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice, 2 (bad-good, answer: B)' + reference: '' + 757da679-3253-463c-9ad3-79a1cd8ae8bf: !Template + answer_choices: B ||| A + id: 757da679-3253-463c-9ad3-79a1cd8ae8bf + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice 1 (good-bad, answer: A)' + reference: '' + 8472a0ce-e98b-4356-a536-bfe2be2554f7: !Template + answer_choices: null + id: 8472a0ce-e98b-4356-a536-bfe2be2554f7 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2_2 + reference: '' + 84c781f2-77aa-4cc4-9577-5084ce3672ff: !Template + answer_choices: null + id: 84c781f2-77aa-4cc4-9577-5084ce3672ff + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B_2 + reference: '' + 8b592dd2-2512-49d2-aa89-beb7ec3d0e6f: !Template + answer_choices: A ||| B + id: 8b592dd2-2512-49d2-aa89-beb7ec3d0e6f + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 2 (bad-good, answer: B)' + reference: '' + ba9dc361-eaaf-41a7-af6d-f3930c3446fb: !Template + answer_choices: null + id: ba9dc361-eaaf-41a7-af6d-f3930c3446fb + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B + reference: '' + ec108cab-4df0-48ce-bd35-8a5e13edc4a3: !Template + answer_choices: null + id: ec108cab-4df0-48ce-bd35-8a5e13edc4a3 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2 + reference: '' + f5c8484d-05f8-4ea7-a629-5f746b7d2c90: !Template + answer_choices: Yes ||| No + id: f5c8484d-05f8-4ea7-a629-5f746b7d2c90 + jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' diff --git a/promptsource/templates/blimp/irregular_past_participle_adjectives/templates.yaml b/promptsource/templates/blimp/irregular_past_participle_adjectives/templates.yaml new file mode 100644 index 000000000..81e3f91d6 --- /dev/null +++ b/promptsource/templates/blimp/irregular_past_participle_adjectives/templates.yaml @@ -0,0 +1,235 @@ +dataset: blimp +subset: irregular_past_participle_adjectives +templates: + 1a72c9d4-42eb-4967-9737-d8e98f3525b6: !Template + answer_choices: A ||| B + id: 1a72c9d4-42eb-4967-9737-d8e98f3525b6 + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 1 (good-bad, answer: A)' + reference: '' + 24aabf5a-7a69-4cc7-b498-10342a680928: !Template + answer_choices: B ||| A + id: 24aabf5a-7a69-4cc7-b498-10342a680928 + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice 1 (good-bad, answer: A)' + reference: '' + 3d753b64-903d-4c1c-97ab-aec16858456f: !Template + answer_choices: null + id: 3d753b64-903d-4c1c-97ab-aec16858456f + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2 + reference: '' + 9f7e3bfa-603c-4426-9f4a-2a95d6681e9d: !Template + answer_choices: A ||| B + id: 9f7e3bfa-603c-4426-9f4a-2a95d6681e9d + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 2 (bad-good, answer: B)' + reference: '' + cacb755f-525a-4d9c-bfd2-0ff72bdfde36: !Template + answer_choices: null + id: cacb755f-525a-4d9c-bfd2-0ff72bdfde36 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B_2 + reference: '' + d64cf1fe-9435-40fc-b1b9-a7375f75940d: !Template + answer_choices: Yes ||| No + id: d64cf1fe-9435-40fc-b1b9-a7375f75940d + jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' + df8909c4-d785-436e-8b15-d60bccdd8999: !Template + answer_choices: null + id: df8909c4-d785-436e-8b15-d60bccdd8999 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B + reference: '' + e4101a58-a7f8-4df5-8f88-c92b2efd47c2: !Template + answer_choices: Yes ||| No + id: e4101a58-a7f8-4df5-8f88-c92b2efd47c2 + jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' + ee6ebb5a-49a2-4e9e-b34e-8f95607dc9d1: !Template + answer_choices: B ||| A + id: ee6ebb5a-49a2-4e9e-b34e-8f95607dc9d1 + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice, 2 (bad-good, answer: B)' + reference: '' + fbb1facd-f57b-47b7-82b3-e107bbc20e65: !Template + answer_choices: null + id: fbb1facd-f57b-47b7-82b3-e107bbc20e65 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2_2 + reference: '' diff --git a/promptsource/templates/blimp/irregular_past_participle_verbs/templates.yaml b/promptsource/templates/blimp/irregular_past_participle_verbs/templates.yaml new file mode 100644 index 000000000..bdf0265a1 --- /dev/null +++ b/promptsource/templates/blimp/irregular_past_participle_verbs/templates.yaml @@ -0,0 +1,235 @@ +dataset: blimp +subset: irregular_past_participle_verbs +templates: + 28aa62d4-83b4-42d8-b090-0557c0d7132a: !Template + answer_choices: Yes ||| No + id: 28aa62d4-83b4-42d8-b090-0557c0d7132a + jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' + 3c661493-e0ce-4469-a1c6-672172ad4362: !Template + answer_choices: B ||| A + id: 3c661493-e0ce-4469-a1c6-672172ad4362 + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice, 2 (bad-good, answer: B)' + reference: '' + 4904ebe5-2da9-40b7-bfb7-10a6775a8eb0: !Template + answer_choices: Yes ||| No + id: 4904ebe5-2da9-40b7-bfb7-10a6775a8eb0 + jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' + 51c3e1c5-02c8-4ede-9766-d8dd8b24ce68: !Template + answer_choices: null + id: 51c3e1c5-02c8-4ede-9766-d8dd8b24ce68 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B_2 + reference: '' + 7c96cdf3-38a3-40ce-a3d1-c747af08902c: !Template + answer_choices: B ||| A + id: 7c96cdf3-38a3-40ce-a3d1-c747af08902c + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice 1 (good-bad, answer: A)' + reference: '' + 8d7fd6ac-98bc-46b4-99ea-2582384a89e7: !Template + answer_choices: null + id: 8d7fd6ac-98bc-46b4-99ea-2582384a89e7 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2 + reference: '' + 93989e52-d946-4bb2-b0c3-63cbf8971333: !Template + answer_choices: null + id: 93989e52-d946-4bb2-b0c3-63cbf8971333 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B + reference: '' + b57f9e72-5ed7-4702-810b-bbe5e0fe17d6: !Template + answer_choices: A ||| B + id: b57f9e72-5ed7-4702-810b-bbe5e0fe17d6 + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 1 (good-bad, answer: A)' + reference: '' + b74c602d-5a1f-49da-9441-5bce4187e0b6: !Template + answer_choices: A ||| B + id: b74c602d-5a1f-49da-9441-5bce4187e0b6 + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 2 (bad-good, answer: B)' + reference: '' + d016c1e6-3545-4825-bae9-9cc25b6e14f1: !Template + answer_choices: null + id: d016c1e6-3545-4825-bae9-9cc25b6e14f1 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2_2 + reference: '' diff --git a/promptsource/templates/blimp/irregular_plural_subject_verb_agreement_1/templates.yaml b/promptsource/templates/blimp/irregular_plural_subject_verb_agreement_1/templates.yaml new file mode 100644 index 000000000..47ff76103 --- /dev/null +++ b/promptsource/templates/blimp/irregular_plural_subject_verb_agreement_1/templates.yaml @@ -0,0 +1,235 @@ +dataset: blimp +subset: irregular_plural_subject_verb_agreement_1 +templates: + 06e6d299-ddd2-45e6-8ec3-3c0bf9774323: !Template + answer_choices: A ||| B + id: 06e6d299-ddd2-45e6-8ec3-3c0bf9774323 + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 1 (good-bad, answer: A)' + reference: '' + 212670be-9b84-4386-beec-bf09dbb31161: !Template + answer_choices: Yes ||| No + id: 212670be-9b84-4386-beec-bf09dbb31161 + jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' + 3de6f05b-6c60-48c1-b91a-b10b9dadb022: !Template + answer_choices: A ||| B + id: 3de6f05b-6c60-48c1-b91a-b10b9dadb022 + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 2 (bad-good, answer: B)' + reference: '' + 737cba97-50ce-4f61-9882-fd98e361a3dd: !Template + answer_choices: null + id: 737cba97-50ce-4f61-9882-fd98e361a3dd + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2 + reference: '' + 784efb6b-2959-48df-b9a7-8ccad19b1f11: !Template + answer_choices: B ||| A + id: 784efb6b-2959-48df-b9a7-8ccad19b1f11 + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice 1 (good-bad, answer: A)' + reference: '' + 9a7885e6-9381-4d96-8614-3e3307627e85: !Template + answer_choices: null + id: 9a7885e6-9381-4d96-8614-3e3307627e85 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B + reference: '' + b8ece796-4f9c-4176-b809-319c85ba8083: !Template + answer_choices: B ||| A + id: b8ece796-4f9c-4176-b809-319c85ba8083 + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice, 2 (bad-good, answer: B)' + reference: '' + d20bcffc-1bd0-4058-8101-c35cdafde31a: !Template + answer_choices: null + id: d20bcffc-1bd0-4058-8101-c35cdafde31a + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2_2 + reference: '' + e2c2996c-d9c9-4085-b945-953ba5677520: !Template + answer_choices: Yes ||| No + id: e2c2996c-d9c9-4085-b945-953ba5677520 + jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' + f6733076-3461-4522-b5cd-38cc50469c48: !Template + answer_choices: null + id: f6733076-3461-4522-b5cd-38cc50469c48 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B_2 + reference: '' diff --git a/promptsource/templates/blimp/irregular_plural_subject_verb_agreement_2/templates.yaml b/promptsource/templates/blimp/irregular_plural_subject_verb_agreement_2/templates.yaml new file mode 100644 index 000000000..df3e571d9 --- /dev/null +++ b/promptsource/templates/blimp/irregular_plural_subject_verb_agreement_2/templates.yaml @@ -0,0 +1,235 @@ +dataset: blimp +subset: irregular_plural_subject_verb_agreement_2 +templates: + 0cb636cc-79c8-41d5-830c-c5b6e911152c: !Template + answer_choices: A ||| B + id: 0cb636cc-79c8-41d5-830c-c5b6e911152c + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 1 (good-bad, answer: A)' + reference: '' + 2309b89d-aa5b-42e1-a3a0-18df116d1b89: !Template + answer_choices: Yes ||| No + id: 2309b89d-aa5b-42e1-a3a0-18df116d1b89 + jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' + 5a5953f1-1122-4c42-9013-a86507bd70c6: !Template + answer_choices: A ||| B + id: 5a5953f1-1122-4c42-9013-a86507bd70c6 + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 2 (bad-good, answer: B)' + reference: '' + 602abc32-08ac-4f01-a290-7c334c5b0275: !Template + answer_choices: null + id: 602abc32-08ac-4f01-a290-7c334c5b0275 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B_2 + reference: '' + 6a067177-3527-47b1-9667-852d967591c0: !Template + answer_choices: Yes ||| No + id: 6a067177-3527-47b1-9667-852d967591c0 + jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' + 923eb5b8-43e3-472c-b5bb-70f3422bec0a: !Template + answer_choices: null + id: 923eb5b8-43e3-472c-b5bb-70f3422bec0a + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2_2 + reference: '' + 99dd4962-0a8b-4f2e-82dd-41180a15e3ca: !Template + answer_choices: null + id: 99dd4962-0a8b-4f2e-82dd-41180a15e3ca + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B + reference: '' + b06e97b2-0d79-4e16-96da-444683e45b71: !Template + answer_choices: B ||| A + id: b06e97b2-0d79-4e16-96da-444683e45b71 + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice, 2 (bad-good, answer: B)' + reference: '' + b4a15970-4849-4b6b-b3c0-a93a8181b385: !Template + answer_choices: B ||| A + id: b4a15970-4849-4b6b-b3c0-a93a8181b385 + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice 1 (good-bad, answer: A)' + reference: '' + d9e70db4-80d1-47af-a029-92082aa0739f: !Template + answer_choices: null + id: d9e70db4-80d1-47af-a029-92082aa0739f + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2 + reference: '' diff --git a/promptsource/templates/blimp/left_branch_island_echo_question/templates.yaml b/promptsource/templates/blimp/left_branch_island_echo_question/templates.yaml new file mode 100644 index 000000000..c518ed84e --- /dev/null +++ b/promptsource/templates/blimp/left_branch_island_echo_question/templates.yaml @@ -0,0 +1,235 @@ +dataset: blimp +subset: left_branch_island_echo_question +templates: + 2a0d77a0-689c-4871-aabb-658c8be08dbb: !Template + answer_choices: null + id: 2a0d77a0-689c-4871-aabb-658c8be08dbb + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B_2 + reference: '' + 59cce349-5011-46f2-9403-e641d5eea89e: !Template + answer_choices: B ||| A + id: 59cce349-5011-46f2-9403-e641d5eea89e + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice, 2 (bad-good, answer: B)' + reference: '' + 5c0750d9-45fa-45f0-92ab-82916134f126: !Template + answer_choices: Yes ||| No + id: 5c0750d9-45fa-45f0-92ab-82916134f126 + jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' + 70131810-2398-496a-903a-b518f2193f38: !Template + answer_choices: null + id: 70131810-2398-496a-903a-b518f2193f38 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2 + reference: '' + 7893c399-c3dd-4515-baa1-59dd8c28c265: !Template + answer_choices: null + id: 7893c399-c3dd-4515-baa1-59dd8c28c265 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B + reference: '' + 8a09370c-d922-416c-b0b5-0cb8cfeddef5: !Template + answer_choices: B ||| A + id: 8a09370c-d922-416c-b0b5-0cb8cfeddef5 + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice 1 (good-bad, answer: A)' + reference: '' + 918d5796-3bd1-4492-96d4-c63c6ee418b0: !Template + answer_choices: A ||| B + id: 918d5796-3bd1-4492-96d4-c63c6ee418b0 + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 1 (good-bad, answer: A)' + reference: '' + acee5cf1-dd6c-41e0-9b7d-d756893ceb21: !Template + answer_choices: Yes ||| No + id: acee5cf1-dd6c-41e0-9b7d-d756893ceb21 + jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' + d69689cf-0926-4ac8-9b77-8e8d88e88747: !Template + answer_choices: null + id: d69689cf-0926-4ac8-9b77-8e8d88e88747 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2_2 + reference: '' + ff26de8b-4911-47f3-bc96-777c14320e82: !Template + answer_choices: A ||| B + id: ff26de8b-4911-47f3-bc96-777c14320e82 + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 2 (bad-good, answer: B)' + reference: '' diff --git a/promptsource/templates/blimp/left_branch_island_simple_question/templates.yaml b/promptsource/templates/blimp/left_branch_island_simple_question/templates.yaml new file mode 100644 index 000000000..9d820fd38 --- /dev/null +++ b/promptsource/templates/blimp/left_branch_island_simple_question/templates.yaml @@ -0,0 +1,235 @@ +dataset: blimp +subset: left_branch_island_simple_question +templates: + 0b748807-1d95-4764-8b42-8b45bb26ddeb: !Template + answer_choices: null + id: 0b748807-1d95-4764-8b42-8b45bb26ddeb + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B + reference: '' + 11975bac-b906-426d-ac75-1f00d6556a8f: !Template + answer_choices: null + id: 11975bac-b906-426d-ac75-1f00d6556a8f + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2 + reference: '' + 28ee916d-b379-4757-a016-74a18e36cf01: !Template + answer_choices: A ||| B + id: 28ee916d-b379-4757-a016-74a18e36cf01 + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 1 (good-bad, answer: A)' + reference: '' + 3cdd553b-aae7-4a6a-9086-13b11ee1793e: !Template + answer_choices: B ||| A + id: 3cdd553b-aae7-4a6a-9086-13b11ee1793e + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice 1 (good-bad, answer: A)' + reference: '' + 48640caf-c52c-4c14-985e-592ca3ddab7d: !Template + answer_choices: null + id: 48640caf-c52c-4c14-985e-592ca3ddab7d + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2_2 + reference: '' + 4c2541de-62eb-4e42-a16a-043ea5b4a592: !Template + answer_choices: Yes ||| No + id: 4c2541de-62eb-4e42-a16a-043ea5b4a592 + jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' + 62267fed-2552-489f-9426-659c797d89c2: !Template + answer_choices: A ||| B + id: 62267fed-2552-489f-9426-659c797d89c2 + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 2 (bad-good, answer: B)' + reference: '' + 932e58d7-cee2-490e-bc82-a94c9e1e81c1: !Template + answer_choices: null + id: 932e58d7-cee2-490e-bc82-a94c9e1e81c1 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B_2 + reference: '' + b621a773-8012-40ac-8302-98e930395200: !Template + answer_choices: Yes ||| No + id: b621a773-8012-40ac-8302-98e930395200 + jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' + e6c7fbb0-65ed-4faa-9d47-0f980535c75d: !Template + answer_choices: B ||| A + id: e6c7fbb0-65ed-4faa-9d47-0f980535c75d + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice, 2 (bad-good, answer: B)' + reference: '' diff --git a/promptsource/templates/blimp/matrix_question_npi_licensor_present/templates.yaml b/promptsource/templates/blimp/matrix_question_npi_licensor_present/templates.yaml new file mode 100644 index 000000000..f81c128cc --- /dev/null +++ b/promptsource/templates/blimp/matrix_question_npi_licensor_present/templates.yaml @@ -0,0 +1,235 @@ +dataset: blimp +subset: matrix_question_npi_licensor_present +templates: + 2ee4a89b-4a80-4639-8a6f-7798141ac45b: !Template + answer_choices: B ||| A + id: 2ee4a89b-4a80-4639-8a6f-7798141ac45b + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice, 2 (bad-good, answer: B)' + reference: '' + 353a20b7-daf7-43c4-8a9d-356a039b8422: !Template + answer_choices: Yes ||| No + id: 353a20b7-daf7-43c4-8a9d-356a039b8422 + jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' + 38467926-4540-4d7d-853d-fd7c4cd31519: !Template + answer_choices: A ||| B + id: 38467926-4540-4d7d-853d-fd7c4cd31519 + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 2 (bad-good, answer: B)' + reference: '' + 41c29d52-e4e5-4ec7-86f9-d94e06d72a3d: !Template + answer_choices: A ||| B + id: 41c29d52-e4e5-4ec7-86f9-d94e06d72a3d + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 1 (good-bad, answer: A)' + reference: '' + 514cb156-b57e-43b0-bc58-ae6439487c7d: !Template + answer_choices: Yes ||| No + id: 514cb156-b57e-43b0-bc58-ae6439487c7d + jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' + 75c48a8a-96ab-4fe6-b84f-a018e29a6e6a: !Template + answer_choices: null + id: 75c48a8a-96ab-4fe6-b84f-a018e29a6e6a + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2 + reference: '' + 80972308-0476-4ea4-9262-8d3ea7fcbfac: !Template + answer_choices: null + id: 80972308-0476-4ea4-9262-8d3ea7fcbfac + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B_2 + reference: '' + b41aaa31-e18d-4168-a419-1ad39b503ba7: !Template + answer_choices: null + id: b41aaa31-e18d-4168-a419-1ad39b503ba7 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2_2 + reference: '' + cfb7e5a0-9a33-49c0-bfeb-9db76af3174a: !Template + answer_choices: B ||| A + id: cfb7e5a0-9a33-49c0-bfeb-9db76af3174a + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice 1 (good-bad, answer: A)' + reference: '' + f63c9f36-330c-4153-9dc4-79ce590de043: !Template + answer_choices: null + id: f63c9f36-330c-4153-9dc4-79ce590de043 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B + reference: '' diff --git a/promptsource/templates/blimp/npi_present_1/templates.yaml b/promptsource/templates/blimp/npi_present_1/templates.yaml new file mode 100644 index 000000000..8a1d32bb4 --- /dev/null +++ b/promptsource/templates/blimp/npi_present_1/templates.yaml @@ -0,0 +1,235 @@ +dataset: blimp +subset: npi_present_1 +templates: + 168e8577-4d03-483c-8c6d-a03163582af3: !Template + answer_choices: Yes ||| No + id: 168e8577-4d03-483c-8c6d-a03163582af3 + jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' + 3bc65cb7-9340-4d86-b8e0-523bb7db0291: !Template + answer_choices: B ||| A + id: 3bc65cb7-9340-4d86-b8e0-523bb7db0291 + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice, 2 (bad-good, answer: B)' + reference: '' + 514edf8b-5e8d-48d5-9176-178271e6be8e: !Template + answer_choices: A ||| B + id: 514edf8b-5e8d-48d5-9176-178271e6be8e + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 1 (good-bad, answer: A)' + reference: '' + 545cd705-4763-45bf-8cc3-5a5e1d51f6fe: !Template + answer_choices: B ||| A + id: 545cd705-4763-45bf-8cc3-5a5e1d51f6fe + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice 1 (good-bad, answer: A)' + reference: '' + 58c85c61-1fc3-4b00-9e8b-4f6a24b0f9c8: !Template + answer_choices: Yes ||| No + id: 58c85c61-1fc3-4b00-9e8b-4f6a24b0f9c8 + jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' + 5b2a209b-90c6-4d0e-b08f-ddf5e2755b8b: !Template + answer_choices: A ||| B + id: 5b2a209b-90c6-4d0e-b08f-ddf5e2755b8b + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 2 (bad-good, answer: B)' + reference: '' + 6cb52fe9-6c71-414f-ac75-b30e1f60a215: !Template + answer_choices: null + id: 6cb52fe9-6c71-414f-ac75-b30e1f60a215 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2 + reference: '' + 90791114-21bb-4a0e-8fc9-5242aa57b3d4: !Template + answer_choices: null + id: 90791114-21bb-4a0e-8fc9-5242aa57b3d4 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B + reference: '' + ad82524e-20ad-4114-a372-77474edfe14f: !Template + answer_choices: null + id: ad82524e-20ad-4114-a372-77474edfe14f + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B_2 + reference: '' + b05426a4-6323-43a2-a67a-85f3a15731c7: !Template + answer_choices: null + id: b05426a4-6323-43a2-a67a-85f3a15731c7 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2_2 + reference: '' diff --git a/promptsource/templates/blimp/npi_present_2/templates.yaml b/promptsource/templates/blimp/npi_present_2/templates.yaml new file mode 100644 index 000000000..48c61629d --- /dev/null +++ b/promptsource/templates/blimp/npi_present_2/templates.yaml @@ -0,0 +1,235 @@ +dataset: blimp +subset: npi_present_2 +templates: + 0ae337be-dac3-41a7-a9c0-96e702a01503: !Template + answer_choices: Yes ||| No + id: 0ae337be-dac3-41a7-a9c0-96e702a01503 + jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' + 169fc98d-bd90-4b90-8e90-703372dfdab8: !Template + answer_choices: Yes ||| No + id: 169fc98d-bd90-4b90-8e90-703372dfdab8 + jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' + 2c7a4a20-601b-47a7-af37-66d83e66cdba: !Template + answer_choices: null + id: 2c7a4a20-601b-47a7-af37-66d83e66cdba + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2_2 + reference: '' + 2cdc825c-ca4d-4758-9cfd-a20bfac9aef6: !Template + answer_choices: A ||| B + id: 2cdc825c-ca4d-4758-9cfd-a20bfac9aef6 + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 2 (bad-good, answer: B)' + reference: '' + 70b7fcdb-9e93-411f-80c6-48ad42daf1b0: !Template + answer_choices: B ||| A + id: 70b7fcdb-9e93-411f-80c6-48ad42daf1b0 + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice 1 (good-bad, answer: A)' + reference: '' + 88beaf5b-223c-4449-8267-82624ce983c7: !Template + answer_choices: B ||| A + id: 88beaf5b-223c-4449-8267-82624ce983c7 + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice, 2 (bad-good, answer: B)' + reference: '' + 95d9a3b8-722a-47b5-add4-5ecc1f9bf65b: !Template + answer_choices: null + id: 95d9a3b8-722a-47b5-add4-5ecc1f9bf65b + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B_2 + reference: '' + b2e28836-c158-4436-bf6b-44d8eddd8dcb: !Template + answer_choices: A ||| B + id: b2e28836-c158-4436-bf6b-44d8eddd8dcb + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 1 (good-bad, answer: A)' + reference: '' + e33a6c12-f64b-4a2e-bffe-c67ac4248ca7: !Template + answer_choices: null + id: e33a6c12-f64b-4a2e-bffe-c67ac4248ca7 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2 + reference: '' + f9aa6f26-0a21-41a4-8f46-b107647c7689: !Template + answer_choices: null + id: f9aa6f26-0a21-41a4-8f46-b107647c7689 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B + reference: '' diff --git a/promptsource/templates/blimp/only_npi_licensor_present/templates.yaml b/promptsource/templates/blimp/only_npi_licensor_present/templates.yaml new file mode 100644 index 000000000..48ad5fe41 --- /dev/null +++ b/promptsource/templates/blimp/only_npi_licensor_present/templates.yaml @@ -0,0 +1,235 @@ +dataset: blimp +subset: only_npi_licensor_present +templates: + 2b0af143-b265-4f52-81df-ea1fe567628d: !Template + answer_choices: A ||| B + id: 2b0af143-b265-4f52-81df-ea1fe567628d + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 2 (bad-good, answer: B)' + reference: '' + 66751e2b-058d-49ce-b3a9-0089ffe1c019: !Template + answer_choices: null + id: 66751e2b-058d-49ce-b3a9-0089ffe1c019 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2 + reference: '' + 74011910-24b0-44fd-b4ef-f11dc338257e: !Template + answer_choices: B ||| A + id: 74011910-24b0-44fd-b4ef-f11dc338257e + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice 1 (good-bad, answer: A)' + reference: '' + a4a13d4f-9e72-432c-b43a-a0d9efe9c5c3: !Template + answer_choices: Yes ||| No + id: a4a13d4f-9e72-432c-b43a-a0d9efe9c5c3 + jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' + aa839e00-65d2-4c07-afcd-9abfcfe010ff: !Template + answer_choices: null + id: aa839e00-65d2-4c07-afcd-9abfcfe010ff + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B + reference: '' + b7e66774-2b87-40d4-ae17-38ebeec34539: !Template + answer_choices: null + id: b7e66774-2b87-40d4-ae17-38ebeec34539 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B_2 + reference: '' + e0922693-1a3d-444e-b0da-7e21d6349826: !Template + answer_choices: A ||| B + id: e0922693-1a3d-444e-b0da-7e21d6349826 + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 1 (good-bad, answer: A)' + reference: '' + e117359f-4d81-407d-a8fd-ecba96ccb2ad: !Template + answer_choices: B ||| A + id: e117359f-4d81-407d-a8fd-ecba96ccb2ad + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice, 2 (bad-good, answer: B)' + reference: '' + e72f656b-f91d-4147-a2f9-5e6f81f27a41: !Template + answer_choices: Yes ||| No + id: e72f656b-f91d-4147-a2f9-5e6f81f27a41 + jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' + fb777ae8-278d-447f-a291-a0b76d606f05: !Template + answer_choices: null + id: fb777ae8-278d-447f-a291-a0b76d606f05 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2_2 + reference: '' diff --git a/promptsource/templates/blimp/only_npi_scope/templates.yaml b/promptsource/templates/blimp/only_npi_scope/templates.yaml new file mode 100644 index 000000000..b325f45d0 --- /dev/null +++ b/promptsource/templates/blimp/only_npi_scope/templates.yaml @@ -0,0 +1,235 @@ +dataset: blimp +subset: only_npi_scope +templates: + 0a230e5d-c5f6-44b6-9657-2d0f058b1cc9: !Template + answer_choices: B ||| A + id: 0a230e5d-c5f6-44b6-9657-2d0f058b1cc9 + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice, 2 (bad-good, answer: B)' + reference: '' + 4d9457ba-b135-4733-88a0-07424781e528: !Template + answer_choices: Yes ||| No + id: 4d9457ba-b135-4733-88a0-07424781e528 + jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' + 64de70aa-ecac-4a2c-8c74-da2347a3dfe1: !Template + answer_choices: A ||| B + id: 64de70aa-ecac-4a2c-8c74-da2347a3dfe1 + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 1 (good-bad, answer: A)' + reference: '' + 7aed9ddb-0797-4ce8-9669-ae7e9fe96416: !Template + answer_choices: null + id: 7aed9ddb-0797-4ce8-9669-ae7e9fe96416 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2_2 + reference: '' + 935da619-659d-4a9f-84ff-3701a43c2eac: !Template + answer_choices: null + id: 935da619-659d-4a9f-84ff-3701a43c2eac + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2 + reference: '' + 94b50572-2a21-450b-96d8-24ab3a9e8edf: !Template + answer_choices: Yes ||| No + id: 94b50572-2a21-450b-96d8-24ab3a9e8edf + jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' + a0ae522c-bf49-49c3-8acd-a32a4f7c8086: !Template + answer_choices: A ||| B + id: a0ae522c-bf49-49c3-8acd-a32a4f7c8086 + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 2 (bad-good, answer: B)' + reference: '' + b53108b5-95a3-4b3e-99ee-d9f4dc9c0358: !Template + answer_choices: B ||| A + id: b53108b5-95a3-4b3e-99ee-d9f4dc9c0358 + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice 1 (good-bad, answer: A)' + reference: '' + b8c83d50-3d52-4f17-a173-f9a1588bcb25: !Template + answer_choices: null + id: b8c83d50-3d52-4f17-a173-f9a1588bcb25 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B + reference: '' + cc03dab5-2b6f-4718-9b06-063a4cf3891f: !Template + answer_choices: null + id: cc03dab5-2b6f-4718-9b06-063a4cf3891f + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B_2 + reference: '' diff --git a/promptsource/templates/blimp/passive_1/templates.yaml b/promptsource/templates/blimp/passive_1/templates.yaml new file mode 100644 index 000000000..f52ba25aa --- /dev/null +++ b/promptsource/templates/blimp/passive_1/templates.yaml @@ -0,0 +1,235 @@ +dataset: blimp +subset: passive_1 +templates: + 009eade8-706c-4d35-a3c4-aa1d7abf1c44: !Template + answer_choices: null + id: 009eade8-706c-4d35-a3c4-aa1d7abf1c44 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B + reference: '' + 423ae75a-bf5b-432d-a7b1-886787b8e042: !Template + answer_choices: B ||| A + id: 423ae75a-bf5b-432d-a7b1-886787b8e042 + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice 1 (good-bad, answer: A)' + reference: '' + 6e5547b8-ee1d-4c88-8710-8d0350c0fec4: !Template + answer_choices: B ||| A + id: 6e5547b8-ee1d-4c88-8710-8d0350c0fec4 + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice, 2 (bad-good, answer: B)' + reference: '' + 880be2b4-7fd4-482f-81e7-06224b2348b3: !Template + answer_choices: A ||| B + id: 880be2b4-7fd4-482f-81e7-06224b2348b3 + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 1 (good-bad, answer: A)' + reference: '' + 96f50bba-2d1c-45f8-bc4d-4e525861070f: !Template + answer_choices: null + id: 96f50bba-2d1c-45f8-bc4d-4e525861070f + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2 + reference: '' + 9a67d7ba-dc0d-4365-adc0-ff49fcfc0d0c: !Template + answer_choices: Yes ||| No + id: 9a67d7ba-dc0d-4365-adc0-ff49fcfc0d0c + jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' + b1f1f897-fd46-4862-bf43-dea47271b37f: !Template + answer_choices: null + id: b1f1f897-fd46-4862-bf43-dea47271b37f + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2_2 + reference: '' + c3d01433-ddba-4b7c-bd60-18cef6237a42: !Template + answer_choices: Yes ||| No + id: c3d01433-ddba-4b7c-bd60-18cef6237a42 + jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' + db7ddd1e-3f66-414f-9b02-2254154a6103: !Template + answer_choices: A ||| B + id: db7ddd1e-3f66-414f-9b02-2254154a6103 + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 2 (bad-good, answer: B)' + reference: '' + f28e549e-34bd-4a7f-af29-570dd110cac0: !Template + answer_choices: null + id: f28e549e-34bd-4a7f-af29-570dd110cac0 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B_2 + reference: '' diff --git a/promptsource/templates/blimp/passive_2/templates.yaml b/promptsource/templates/blimp/passive_2/templates.yaml new file mode 100644 index 000000000..fe0c2db2e --- /dev/null +++ b/promptsource/templates/blimp/passive_2/templates.yaml @@ -0,0 +1,235 @@ +dataset: blimp +subset: passive_2 +templates: + 0d59ed08-a317-4672-ade2-3ef1ef4eda64: !Template + answer_choices: B ||| A + id: 0d59ed08-a317-4672-ade2-3ef1ef4eda64 + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice 1 (good-bad, answer: A)' + reference: '' + 196bba57-312d-4c64-b56d-ce1dac736772: !Template + answer_choices: null + id: 196bba57-312d-4c64-b56d-ce1dac736772 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2_2 + reference: '' + 2ecc2dca-9b3d-4340-85ec-9ed8ae25714a: !Template + answer_choices: null + id: 2ecc2dca-9b3d-4340-85ec-9ed8ae25714a + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B_2 + reference: '' + 3cc44323-551c-464d-8222-dbc8dcad1873: !Template + answer_choices: A ||| B + id: 3cc44323-551c-464d-8222-dbc8dcad1873 + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 2 (bad-good, answer: B)' + reference: '' + 58684b10-f0a2-484c-8bb3-9f565623474a: !Template + answer_choices: B ||| A + id: 58684b10-f0a2-484c-8bb3-9f565623474a + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice, 2 (bad-good, answer: B)' + reference: '' + 675e20e3-aabe-4254-9df0-a3b24b2f4adf: !Template + answer_choices: A ||| B + id: 675e20e3-aabe-4254-9df0-a3b24b2f4adf + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 1 (good-bad, answer: A)' + reference: '' + 69d66f1a-eaed-40da-b847-3e90e61b1418: !Template + answer_choices: Yes ||| No + id: 69d66f1a-eaed-40da-b847-3e90e61b1418 + jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' + a1e17a9c-ad8a-4dae-a022-161084f75a97: !Template + answer_choices: Yes ||| No + id: a1e17a9c-ad8a-4dae-a022-161084f75a97 + jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' + c87bad0e-bf1f-4f5a-9e82-9f78a83493df: !Template + answer_choices: null + id: c87bad0e-bf1f-4f5a-9e82-9f78a83493df + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B + reference: '' + d7cc6c34-590e-4f31-8a4c-9ddb9ceb824b: !Template + answer_choices: null + id: d7cc6c34-590e-4f31-8a4c-9ddb9ceb824b + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2 + reference: '' diff --git a/promptsource/templates/blimp/principle_A_c_command/templates.yaml b/promptsource/templates/blimp/principle_A_c_command/templates.yaml new file mode 100644 index 000000000..3a55cd532 --- /dev/null +++ b/promptsource/templates/blimp/principle_A_c_command/templates.yaml @@ -0,0 +1,235 @@ +dataset: blimp +subset: principle_A_c_command +templates: + 005e713f-7db1-4e6c-8eff-9d4b430150de: !Template + answer_choices: null + id: 005e713f-7db1-4e6c-8eff-9d4b430150de + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B_2 + reference: '' + 17a6b0f6-e715-489f-9ff3-67b28778d152: !Template + answer_choices: Yes ||| No + id: 17a6b0f6-e715-489f-9ff3-67b28778d152 + jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' + 2d5b5be8-da66-4716-91fc-4a593dc35ec4: !Template + answer_choices: null + id: 2d5b5be8-da66-4716-91fc-4a593dc35ec4 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2 + reference: '' + 4c14aacc-78a7-4afb-b3b9-69da2a9d1d64: !Template + answer_choices: B ||| A + id: 4c14aacc-78a7-4afb-b3b9-69da2a9d1d64 + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice, 2 (bad-good, answer: B)' + reference: '' + 54480a25-a63a-476c-a3cf-edbf9c92278f: !Template + answer_choices: A ||| B + id: 54480a25-a63a-476c-a3cf-edbf9c92278f + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 2 (bad-good, answer: B)' + reference: '' + 65b6ca78-d43c-4350-a03c-fd66c0379a79: !Template + answer_choices: null + id: 65b6ca78-d43c-4350-a03c-fd66c0379a79 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2_2 + reference: '' + 7e773aac-5edf-488b-82a1-1c98b906091d: !Template + answer_choices: A ||| B + id: 7e773aac-5edf-488b-82a1-1c98b906091d + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 1 (good-bad, answer: A)' + reference: '' + a5059bc2-b979-485f-8935-ade9466030e8: !Template + answer_choices: Yes ||| No + id: a5059bc2-b979-485f-8935-ade9466030e8 + jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' + bc47b240-80c8-4c88-a506-8e22fc58af1d: !Template + answer_choices: null + id: bc47b240-80c8-4c88-a506-8e22fc58af1d + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B + reference: '' + fd96b228-90d9-4c99-bb2e-301b63860104: !Template + answer_choices: B ||| A + id: fd96b228-90d9-4c99-bb2e-301b63860104 + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice 1 (good-bad, answer: A)' + reference: '' diff --git a/promptsource/templates/blimp/principle_A_case_1/templates.yaml b/promptsource/templates/blimp/principle_A_case_1/templates.yaml new file mode 100644 index 000000000..561ccd940 --- /dev/null +++ b/promptsource/templates/blimp/principle_A_case_1/templates.yaml @@ -0,0 +1,235 @@ +dataset: blimp +subset: principle_A_case_1 +templates: + 0ff388ad-d085-4f3f-ba26-384408db607b: !Template + answer_choices: Yes ||| No + id: 0ff388ad-d085-4f3f-ba26-384408db607b + jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' + 1006fde0-2bd9-4432-bba2-4edfb104f733: !Template + answer_choices: null + id: 1006fde0-2bd9-4432-bba2-4edfb104f733 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2 + reference: '' + 350577af-e811-4978-9202-9010b6bbfc72: !Template + answer_choices: B ||| A + id: 350577af-e811-4978-9202-9010b6bbfc72 + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice 1 (good-bad, answer: A)' + reference: '' + 61dfec5f-81e1-4932-a9de-a16928c0908d: !Template + answer_choices: null + id: 61dfec5f-81e1-4932-a9de-a16928c0908d + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B + reference: '' + 6849da5f-97d5-4478-a8f5-296f59698925: !Template + answer_choices: null + id: 6849da5f-97d5-4478-a8f5-296f59698925 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2_2 + reference: '' + 9140207f-a54a-45fb-a239-823c4370ac8d: !Template + answer_choices: null + id: 9140207f-a54a-45fb-a239-823c4370ac8d + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B_2 + reference: '' + bd84633c-bc72-4a34-be17-624572b6ac58: !Template + answer_choices: A ||| B + id: bd84633c-bc72-4a34-be17-624572b6ac58 + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 1 (good-bad, answer: A)' + reference: '' + c00c09b8-9962-423f-9778-2af62b981baa: !Template + answer_choices: A ||| B + id: c00c09b8-9962-423f-9778-2af62b981baa + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 2 (bad-good, answer: B)' + reference: '' + cc2ebe4a-51f8-4c8c-998d-3104ed3663f6: !Template + answer_choices: Yes ||| No + id: cc2ebe4a-51f8-4c8c-998d-3104ed3663f6 + jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' + f0479c50-3f02-4881-9e14-a05eaef12181: !Template + answer_choices: B ||| A + id: f0479c50-3f02-4881-9e14-a05eaef12181 + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice, 2 (bad-good, answer: B)' + reference: '' diff --git a/promptsource/templates/blimp/principle_A_case_2/templates.yaml b/promptsource/templates/blimp/principle_A_case_2/templates.yaml new file mode 100644 index 000000000..e25d20370 --- /dev/null +++ b/promptsource/templates/blimp/principle_A_case_2/templates.yaml @@ -0,0 +1,235 @@ +dataset: blimp +subset: principle_A_case_2 +templates: + 1783c375-fcfa-4792-b00a-6968bf2ed883: !Template + answer_choices: null + id: 1783c375-fcfa-4792-b00a-6968bf2ed883 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B_2 + reference: '' + 259259c7-159c-4328-8906-dac38f978b84: !Template + answer_choices: A ||| B + id: 259259c7-159c-4328-8906-dac38f978b84 + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 1 (good-bad, answer: A)' + reference: '' + 30723e9b-dce5-43d9-812a-bb71a3dd5a46: !Template + answer_choices: B ||| A + id: 30723e9b-dce5-43d9-812a-bb71a3dd5a46 + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice, 2 (bad-good, answer: B)' + reference: '' + 6a1b4ed8-80b1-4fce-ad74-4637644ebaba: !Template + answer_choices: Yes ||| No + id: 6a1b4ed8-80b1-4fce-ad74-4637644ebaba + jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' + 73c23d5a-9a8d-4163-bc89-0bd4972e931c: !Template + answer_choices: Yes ||| No + id: 73c23d5a-9a8d-4163-bc89-0bd4972e931c + jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' + 766aa4a1-42e8-476a-a007-c31bc355f800: !Template + answer_choices: A ||| B + id: 766aa4a1-42e8-476a-a007-c31bc355f800 + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 2 (bad-good, answer: B)' + reference: '' + 8375447f-7e6c-4baf-a7e4-3be7f333c1fa: !Template + answer_choices: null + id: 8375447f-7e6c-4baf-a7e4-3be7f333c1fa + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B + reference: '' + b7f484ac-57f1-4e48-ad33-b82bcd5e6d1b: !Template + answer_choices: null + id: b7f484ac-57f1-4e48-ad33-b82bcd5e6d1b + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2_2 + reference: '' + c7875338-345d-4159-ab5e-7378e077b354: !Template + answer_choices: null + id: c7875338-345d-4159-ab5e-7378e077b354 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2 + reference: '' + f99d7eff-f127-42ea-8bad-d573a9ff7178: !Template + answer_choices: B ||| A + id: f99d7eff-f127-42ea-8bad-d573a9ff7178 + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice 1 (good-bad, answer: A)' + reference: '' diff --git a/promptsource/templates/blimp/principle_A_domain_1/templates.yaml b/promptsource/templates/blimp/principle_A_domain_1/templates.yaml new file mode 100644 index 000000000..3f425377a --- /dev/null +++ b/promptsource/templates/blimp/principle_A_domain_1/templates.yaml @@ -0,0 +1,235 @@ +dataset: blimp +subset: principle_A_domain_1 +templates: + 1fd5a2fa-e668-4c8a-9ebe-2957671ad095: !Template + answer_choices: null + id: 1fd5a2fa-e668-4c8a-9ebe-2957671ad095 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2_2 + reference: '' + 23d44482-13c3-420a-981b-5ae39c8898f6: !Template + answer_choices: A ||| B + id: 23d44482-13c3-420a-981b-5ae39c8898f6 + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 2 (bad-good, answer: B)' + reference: '' + 72177185-ab22-4403-8d10-174b680414cf: !Template + answer_choices: B ||| A + id: 72177185-ab22-4403-8d10-174b680414cf + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice, 2 (bad-good, answer: B)' + reference: '' + a9f8b70e-a109-4c8d-95b7-78643fe3509e: !Template + answer_choices: null + id: a9f8b70e-a109-4c8d-95b7-78643fe3509e + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2 + reference: '' + c7447ade-65ac-496b-9e82-0458c177b74f: !Template + answer_choices: null + id: c7447ade-65ac-496b-9e82-0458c177b74f + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B + reference: '' + cdd87a1b-d9c8-44bd-8db3-5974e00d00fe: !Template + answer_choices: Yes ||| No + id: cdd87a1b-d9c8-44bd-8db3-5974e00d00fe + jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' + d4630f14-86de-44a6-832b-7b71eb2e48b2: !Template + answer_choices: Yes ||| No + id: d4630f14-86de-44a6-832b-7b71eb2e48b2 + jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' + d4db4dfe-74df-47d9-870d-65cf6461759b: !Template + answer_choices: null + id: d4db4dfe-74df-47d9-870d-65cf6461759b + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B_2 + reference: '' + ebf0f263-8113-46fa-902a-f8fb710a0bcd: !Template + answer_choices: B ||| A + id: ebf0f263-8113-46fa-902a-f8fb710a0bcd + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice 1 (good-bad, answer: A)' + reference: '' + efc49f08-4b8a-4477-9d51-dfb31a062e56: !Template + answer_choices: A ||| B + id: efc49f08-4b8a-4477-9d51-dfb31a062e56 + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 1 (good-bad, answer: A)' + reference: '' diff --git a/promptsource/templates/blimp/principle_A_domain_2/templates.yaml b/promptsource/templates/blimp/principle_A_domain_2/templates.yaml new file mode 100644 index 000000000..0bbd0d742 --- /dev/null +++ b/promptsource/templates/blimp/principle_A_domain_2/templates.yaml @@ -0,0 +1,235 @@ +dataset: blimp +subset: principle_A_domain_2 +templates: + 04f77863-ded0-408a-b930-69b2ede392aa: !Template + answer_choices: B ||| A + id: 04f77863-ded0-408a-b930-69b2ede392aa + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice 1 (good-bad, answer: A)' + reference: '' + 247b2f62-e586-4edd-9e55-a70a3ba822e3: !Template + answer_choices: null + id: 247b2f62-e586-4edd-9e55-a70a3ba822e3 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2_2 + reference: '' + 3c6ef6c4-2fdb-4e67-80c8-ed54ae9fa001: !Template + answer_choices: A ||| B + id: 3c6ef6c4-2fdb-4e67-80c8-ed54ae9fa001 + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 1 (good-bad, answer: A)' + reference: '' + 55e19004-f66c-421f-92fc-dcc1f3e27381: !Template + answer_choices: null + id: 55e19004-f66c-421f-92fc-dcc1f3e27381 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B_2 + reference: '' + 58d68674-6069-4954-8433-fd62c3569420: !Template + answer_choices: Yes ||| No + id: 58d68674-6069-4954-8433-fd62c3569420 + jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' + 78a22835-3107-4dda-a6eb-a3d347714fdc: !Template + answer_choices: B ||| A + id: 78a22835-3107-4dda-a6eb-a3d347714fdc + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice, 2 (bad-good, answer: B)' + reference: '' + 8a96d69b-d9f3-42cb-bb17-0b4020f47df0: !Template + answer_choices: null + id: 8a96d69b-d9f3-42cb-bb17-0b4020f47df0 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B + reference: '' + 9d4ae77b-6681-4d61-be95-cd4d7318169c: !Template + answer_choices: null + id: 9d4ae77b-6681-4d61-be95-cd4d7318169c + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2 + reference: '' + ae170596-102a-4eb7-8d13-e68acb79c4f6: !Template + answer_choices: A ||| B + id: ae170596-102a-4eb7-8d13-e68acb79c4f6 + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 2 (bad-good, answer: B)' + reference: '' + b83d9f01-6f15-4c93-8562-6a5e20249a05: !Template + answer_choices: Yes ||| No + id: b83d9f01-6f15-4c93-8562-6a5e20249a05 + jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' diff --git a/promptsource/templates/blimp/principle_A_domain_3/templates.yaml b/promptsource/templates/blimp/principle_A_domain_3/templates.yaml new file mode 100644 index 000000000..c62e9f91f --- /dev/null +++ b/promptsource/templates/blimp/principle_A_domain_3/templates.yaml @@ -0,0 +1,235 @@ +dataset: blimp +subset: principle_A_domain_3 +templates: + 0df0c10f-d880-4201-85ec-de46124137e0: !Template + answer_choices: A ||| B + id: 0df0c10f-d880-4201-85ec-de46124137e0 + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 1 (good-bad, answer: A)' + reference: '' + 187b10eb-3242-49d6-b23c-474356347392: !Template + answer_choices: null + id: 187b10eb-3242-49d6-b23c-474356347392 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B + reference: '' + 24daac32-406a-470b-863b-a96326ae120d: !Template + answer_choices: null + id: 24daac32-406a-470b-863b-a96326ae120d + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B_2 + reference: '' + 4d88ab78-315f-4968-a4b2-894b2a551b65: !Template + answer_choices: null + id: 4d88ab78-315f-4968-a4b2-894b2a551b65 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2_2 + reference: '' + a764efa4-32fa-4221-b60a-2b35a0e319a0: !Template + answer_choices: B ||| A + id: a764efa4-32fa-4221-b60a-2b35a0e319a0 + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice, 2 (bad-good, answer: B)' + reference: '' + b6481098-3a97-488c-bb64-67a3a80c9a93: !Template + answer_choices: A ||| B + id: b6481098-3a97-488c-bb64-67a3a80c9a93 + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 2 (bad-good, answer: B)' + reference: '' + cbac2443-7ed6-4ece-834b-eb11a5e329fc: !Template + answer_choices: B ||| A + id: cbac2443-7ed6-4ece-834b-eb11a5e329fc + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice 1 (good-bad, answer: A)' + reference: '' + ce9299b1-7dd3-421e-8e95-fb132dbf80c4: !Template + answer_choices: Yes ||| No + id: ce9299b1-7dd3-421e-8e95-fb132dbf80c4 + jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' + d9fb33e0-fc95-4f82-8777-460cdcc9e353: !Template + answer_choices: null + id: d9fb33e0-fc95-4f82-8777-460cdcc9e353 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2 + reference: '' + efa9b032-4ab1-4772-97d9-38e386b9b499: !Template + answer_choices: Yes ||| No + id: efa9b032-4ab1-4772-97d9-38e386b9b499 + jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' diff --git a/promptsource/templates/blimp/principle_A_reconstruction/templates.yaml b/promptsource/templates/blimp/principle_A_reconstruction/templates.yaml new file mode 100644 index 000000000..63cec742d --- /dev/null +++ b/promptsource/templates/blimp/principle_A_reconstruction/templates.yaml @@ -0,0 +1,235 @@ +dataset: blimp +subset: principle_A_reconstruction +templates: + 576c52f7-ac80-4917-8bce-eb280a64d11e: !Template + answer_choices: A ||| B + id: 576c52f7-ac80-4917-8bce-eb280a64d11e + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 1 (good-bad, answer: A)' + reference: '' + 5e4a07d1-7b3c-4c0f-b4b0-b06dd8917521: !Template + answer_choices: B ||| A + id: 5e4a07d1-7b3c-4c0f-b4b0-b06dd8917521 + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice 1 (good-bad, answer: A)' + reference: '' + 5e9d0209-9b95-44ac-aeea-030796c5a1b4: !Template + answer_choices: A ||| B + id: 5e9d0209-9b95-44ac-aeea-030796c5a1b4 + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 2 (bad-good, answer: B)' + reference: '' + 620f2ec7-b155-4c44-8aaf-10fdbfb21b6d: !Template + answer_choices: null + id: 620f2ec7-b155-4c44-8aaf-10fdbfb21b6d + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2_2 + reference: '' + 84d9a2b5-77bc-4280-936f-e909827fdb3b: !Template + answer_choices: null + id: 84d9a2b5-77bc-4280-936f-e909827fdb3b + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B_2 + reference: '' + 9d6c0bce-f8fd-48b7-b0d6-055f123f6c1c: !Template + answer_choices: null + id: 9d6c0bce-f8fd-48b7-b0d6-055f123f6c1c + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2 + reference: '' + 9fc16a9e-d9bb-4faa-8f93-e18ed334080f: !Template + answer_choices: null + id: 9fc16a9e-d9bb-4faa-8f93-e18ed334080f + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B + reference: '' + a1ac5948-bf3d-481c-9a77-6829bcad634c: !Template + answer_choices: B ||| A + id: a1ac5948-bf3d-481c-9a77-6829bcad634c + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice, 2 (bad-good, answer: B)' + reference: '' + a3f11f84-0b45-41c7-a661-e08cad572723: !Template + answer_choices: Yes ||| No + id: a3f11f84-0b45-41c7-a661-e08cad572723 + jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' + b1b994ed-a369-438e-9ce4-69314c4ea4d5: !Template + answer_choices: Yes ||| No + id: b1b994ed-a369-438e-9ce4-69314c4ea4d5 + jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' diff --git a/promptsource/templates/blimp/regular_plural_subject_verb_agreement_1/templates.yaml b/promptsource/templates/blimp/regular_plural_subject_verb_agreement_1/templates.yaml new file mode 100644 index 000000000..8443cbd01 --- /dev/null +++ b/promptsource/templates/blimp/regular_plural_subject_verb_agreement_1/templates.yaml @@ -0,0 +1,235 @@ +dataset: blimp +subset: regular_plural_subject_verb_agreement_1 +templates: + 09aa27be-636f-4d87-b914-035061a4793d: !Template + answer_choices: Yes ||| No + id: 09aa27be-636f-4d87-b914-035061a4793d + jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' + 349f6fe8-4e6e-40f3-bdba-372ce0ba97bb: !Template + answer_choices: A ||| B + id: 349f6fe8-4e6e-40f3-bdba-372ce0ba97bb + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 2 (bad-good, answer: B)' + reference: '' + 4080b351-4025-4763-9c70-08c6e7afb633: !Template + answer_choices: null + id: 4080b351-4025-4763-9c70-08c6e7afb633 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2_2 + reference: '' + 443abdb9-dede-4bf2-aada-6cc4f81669f3: !Template + answer_choices: A ||| B + id: 443abdb9-dede-4bf2-aada-6cc4f81669f3 + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 1 (good-bad, answer: A)' + reference: '' + 65315901-6f9d-42e9-9e35-48ca2326531e: !Template + answer_choices: null + id: 65315901-6f9d-42e9-9e35-48ca2326531e + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2 + reference: '' + 7f455113-5d3e-405e-88da-0628671e1311: !Template + answer_choices: Yes ||| No + id: 7f455113-5d3e-405e-88da-0628671e1311 + jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' + df32b1f8-cb95-4bc5-a218-91127a52e809: !Template + answer_choices: null + id: df32b1f8-cb95-4bc5-a218-91127a52e809 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B + reference: '' + e5a5ae78-5008-4afa-a306-fbea636a8983: !Template + answer_choices: B ||| A + id: e5a5ae78-5008-4afa-a306-fbea636a8983 + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice 1 (good-bad, answer: A)' + reference: '' + f7c5743d-7dc3-4d30-b148-50698897a2af: !Template + answer_choices: null + id: f7c5743d-7dc3-4d30-b148-50698897a2af + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B_2 + reference: '' + fd31f8b9-923f-4c77-b595-1cd17f52bbb5: !Template + answer_choices: B ||| A + id: fd31f8b9-923f-4c77-b595-1cd17f52bbb5 + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice, 2 (bad-good, answer: B)' + reference: '' diff --git a/promptsource/templates/blimp/regular_plural_subject_verb_agreement_2/templates.yaml b/promptsource/templates/blimp/regular_plural_subject_verb_agreement_2/templates.yaml new file mode 100644 index 000000000..2b6b5c122 --- /dev/null +++ b/promptsource/templates/blimp/regular_plural_subject_verb_agreement_2/templates.yaml @@ -0,0 +1,235 @@ +dataset: blimp +subset: regular_plural_subject_verb_agreement_2 +templates: + 0061188f-179e-4654-96be-13267fe36cc2: !Template + answer_choices: Yes ||| No + id: 0061188f-179e-4654-96be-13267fe36cc2 + jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' + 02710ea9-c648-467f-adf8-828aae786848: !Template + answer_choices: B ||| A + id: 02710ea9-c648-467f-adf8-828aae786848 + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice 1 (good-bad, answer: A)' + reference: '' + 0b908939-2173-4c96-aa68-427092225e12: !Template + answer_choices: A ||| B + id: 0b908939-2173-4c96-aa68-427092225e12 + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 2 (bad-good, answer: B)' + reference: '' + 50549b1a-008d-474e-9129-f8cd9a0ed949: !Template + answer_choices: null + id: 50549b1a-008d-474e-9129-f8cd9a0ed949 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B_2 + reference: '' + 61e70ece-6904-4a2d-b8c5-f3ac8b2df65a: !Template + answer_choices: null + id: 61e70ece-6904-4a2d-b8c5-f3ac8b2df65a + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B + reference: '' + 8091fa97-efb9-4bfd-8d67-4395e694dec2: !Template + answer_choices: B ||| A + id: 8091fa97-efb9-4bfd-8d67-4395e694dec2 + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice, 2 (bad-good, answer: B)' + reference: '' + 904d82e6-4d0f-4edb-9632-a2cc7ad6feea: !Template + answer_choices: Yes ||| No + id: 904d82e6-4d0f-4edb-9632-a2cc7ad6feea + jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' + 9575a77e-f5d4-4963-931a-1629eb831b3e: !Template + answer_choices: null + id: 9575a77e-f5d4-4963-931a-1629eb831b3e + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2 + reference: '' + a5b8bdd4-fe99-423e-b5bc-beaf48ab6f40: !Template + answer_choices: A ||| B + id: a5b8bdd4-fe99-423e-b5bc-beaf48ab6f40 + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 1 (good-bad, answer: A)' + reference: '' + db57f37c-26f5-43d4-889b-39ba6fa18c7a: !Template + answer_choices: null + id: db57f37c-26f5-43d4-889b-39ba6fa18c7a + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2_2 + reference: '' diff --git a/promptsource/templates/blimp/sentential_negation_npi_licensor_present/templates.yaml b/promptsource/templates/blimp/sentential_negation_npi_licensor_present/templates.yaml new file mode 100644 index 000000000..4b7d517ce --- /dev/null +++ b/promptsource/templates/blimp/sentential_negation_npi_licensor_present/templates.yaml @@ -0,0 +1,235 @@ +dataset: blimp +subset: sentential_negation_npi_licensor_present +templates: + 021b598b-9462-4cc4-a845-284e4bdac7c2: !Template + answer_choices: Yes ||| No + id: 021b598b-9462-4cc4-a845-284e4bdac7c2 + jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' + 0f887c2a-2018-4716-a317-1e79f6c610cf: !Template + answer_choices: B ||| A + id: 0f887c2a-2018-4716-a317-1e79f6c610cf + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice, 2 (bad-good, answer: B)' + reference: '' + 28e1631f-5dd1-4018-ba73-f24dc826cb6b: !Template + answer_choices: null + id: 28e1631f-5dd1-4018-ba73-f24dc826cb6b + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B + reference: '' + 596c3ec0-7c0a-45d3-a036-520f6fabbf4f: !Template + answer_choices: A ||| B + id: 596c3ec0-7c0a-45d3-a036-520f6fabbf4f + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 1 (good-bad, answer: A)' + reference: '' + 61b38dd8-a0c9-4661-a8f4-7168441b73cc: !Template + answer_choices: null + id: 61b38dd8-a0c9-4661-a8f4-7168441b73cc + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2 + reference: '' + 683db107-36f5-417a-9e37-d3c3250a24d8: !Template + answer_choices: null + id: 683db107-36f5-417a-9e37-d3c3250a24d8 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B_2 + reference: '' + 83ea33f1-6154-4d29-8829-2a2a193943e0: !Template + answer_choices: A ||| B + id: 83ea33f1-6154-4d29-8829-2a2a193943e0 + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 2 (bad-good, answer: B)' + reference: '' + 8864c1f6-e301-46aa-9a17-6157770e6576: !Template + answer_choices: Yes ||| No + id: 8864c1f6-e301-46aa-9a17-6157770e6576 + jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' + 8af75f75-f09d-4e39-842d-22abf0b5b505: !Template + answer_choices: B ||| A + id: 8af75f75-f09d-4e39-842d-22abf0b5b505 + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice 1 (good-bad, answer: A)' + reference: '' + e9c84f3a-f149-455d-a66e-e33a6196f7f0: !Template + answer_choices: null + id: e9c84f3a-f149-455d-a66e-e33a6196f7f0 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2_2 + reference: '' diff --git a/promptsource/templates/blimp/sentential_negation_npi_scope/templates.yaml b/promptsource/templates/blimp/sentential_negation_npi_scope/templates.yaml new file mode 100644 index 000000000..2572f18ca --- /dev/null +++ b/promptsource/templates/blimp/sentential_negation_npi_scope/templates.yaml @@ -0,0 +1,235 @@ +dataset: blimp +subset: sentential_negation_npi_scope +templates: + 0892df76-cbe9-44d1-9025-003a502c1d64: !Template + answer_choices: Yes ||| No + id: 0892df76-cbe9-44d1-9025-003a502c1d64 + jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' + 0fa3f3cd-7ba5-4ab9-a532-c104ec7fe277: !Template + answer_choices: null + id: 0fa3f3cd-7ba5-4ab9-a532-c104ec7fe277 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B + reference: '' + 1b9e3425-945b-40e9-8115-085e793af225: !Template + answer_choices: null + id: 1b9e3425-945b-40e9-8115-085e793af225 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2 + reference: '' + 29a2acf1-61da-4261-a17a-4be7d7bd99b4: !Template + answer_choices: A ||| B + id: 29a2acf1-61da-4261-a17a-4be7d7bd99b4 + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 2 (bad-good, answer: B)' + reference: '' + 366604a0-b413-4bff-9c2b-5ea9692630bb: !Template + answer_choices: A ||| B + id: 366604a0-b413-4bff-9c2b-5ea9692630bb + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 1 (good-bad, answer: A)' + reference: '' + 82f02663-648a-4279-ab89-483df722c26a: !Template + answer_choices: null + id: 82f02663-648a-4279-ab89-483df722c26a + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B_2 + reference: '' + 86261239-0be6-4b17-8460-5ec85f2d1f4b: !Template + answer_choices: B ||| A + id: 86261239-0be6-4b17-8460-5ec85f2d1f4b + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice, 2 (bad-good, answer: B)' + reference: '' + f187f967-4029-4281-8ad7-08ac22d910f2: !Template + answer_choices: null + id: f187f967-4029-4281-8ad7-08ac22d910f2 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2_2 + reference: '' + f2650ead-ff98-4044-8063-1806dff86a80: !Template + answer_choices: B ||| A + id: f2650ead-ff98-4044-8063-1806dff86a80 + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice 1 (good-bad, answer: A)' + reference: '' + f6d72d15-3cbb-4985-9270-462dabeb1cf4: !Template + answer_choices: Yes ||| No + id: f6d72d15-3cbb-4985-9270-462dabeb1cf4 + jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' diff --git a/promptsource/templates/blimp/sentential_subject_island/templates.yaml b/promptsource/templates/blimp/sentential_subject_island/templates.yaml new file mode 100644 index 000000000..cdb2b266e --- /dev/null +++ b/promptsource/templates/blimp/sentential_subject_island/templates.yaml @@ -0,0 +1,235 @@ +dataset: blimp +subset: sentential_subject_island +templates: + 1867a802-40e0-47c0-8c04-8bdac04c9018: !Template + answer_choices: null + id: 1867a802-40e0-47c0-8c04-8bdac04c9018 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2 + reference: '' + 186d1ffa-3bd7-4ba0-bbf2-0dc5d7062ce2: !Template + answer_choices: B ||| A + id: 186d1ffa-3bd7-4ba0-bbf2-0dc5d7062ce2 + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice, 2 (bad-good, answer: B)' + reference: '' + 2494a3da-635f-4491-adc9-d4b7d470dcdf: !Template + answer_choices: null + id: 2494a3da-635f-4491-adc9-d4b7d470dcdf + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B + reference: '' + 28fdc7d7-5d0c-4cb6-8460-4802e0d5004f: !Template + answer_choices: null + id: 28fdc7d7-5d0c-4cb6-8460-4802e0d5004f + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2_2 + reference: '' + 50fe301e-a33c-4e24-bd8e-745902916843: !Template + answer_choices: Yes ||| No + id: 50fe301e-a33c-4e24-bd8e-745902916843 + jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' + 7f4c3371-ba5a-4e42-a5e7-974b8cf19af5: !Template + answer_choices: null + id: 7f4c3371-ba5a-4e42-a5e7-974b8cf19af5 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B_2 + reference: '' + 8daf5406-4c2c-46e4-adff-6827391ba7cd: !Template + answer_choices: Yes ||| No + id: 8daf5406-4c2c-46e4-adff-6827391ba7cd + jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' + 9f9ad12e-1bab-4959-9596-d3478f396787: !Template + answer_choices: A ||| B + id: 9f9ad12e-1bab-4959-9596-d3478f396787 + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 2 (bad-good, answer: B)' + reference: '' + c3109a85-f78d-4dbe-9cca-24a8800fb4cd: !Template + answer_choices: A ||| B + id: c3109a85-f78d-4dbe-9cca-24a8800fb4cd + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 1 (good-bad, answer: A)' + reference: '' + ee405258-203f-47f9-8f85-ca1654bfa1e3: !Template + answer_choices: B ||| A + id: ee405258-203f-47f9-8f85-ca1654bfa1e3 + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice 1 (good-bad, answer: A)' + reference: '' diff --git a/promptsource/templates/blimp/superlative_quantifiers_1/templates.yaml b/promptsource/templates/blimp/superlative_quantifiers_1/templates.yaml new file mode 100644 index 000000000..451854807 --- /dev/null +++ b/promptsource/templates/blimp/superlative_quantifiers_1/templates.yaml @@ -0,0 +1,235 @@ +dataset: blimp +subset: superlative_quantifiers_1 +templates: + 11ccc1f4-2e68-47db-8493-5442f179e952: !Template + answer_choices: B ||| A + id: 11ccc1f4-2e68-47db-8493-5442f179e952 + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice, 2 (bad-good, answer: B)' + reference: '' + 134066c0-60b0-408c-be42-6c594d4c6af5: !Template + answer_choices: Yes ||| No + id: 134066c0-60b0-408c-be42-6c594d4c6af5 + jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' + 1e124c50-9ab6-4c07-b514-4d9cad071a5d: !Template + answer_choices: null + id: 1e124c50-9ab6-4c07-b514-4d9cad071a5d + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B_2 + reference: '' + 1ee846e6-133b-4a31-83c1-de959f7d5491: !Template + answer_choices: null + id: 1ee846e6-133b-4a31-83c1-de959f7d5491 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2_2 + reference: '' + 493c7693-514b-4d3b-91af-6153ecd59061: !Template + answer_choices: A ||| B + id: 493c7693-514b-4d3b-91af-6153ecd59061 + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 1 (good-bad, answer: A)' + reference: '' + 576b9858-f67f-45b8-8cd1-e333737a1f40: !Template + answer_choices: Yes ||| No + id: 576b9858-f67f-45b8-8cd1-e333737a1f40 + jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' + 58ed84ff-a099-47c4-ab63-b57b3e71e726: !Template + answer_choices: null + id: 58ed84ff-a099-47c4-ab63-b57b3e71e726 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B + reference: '' + 80f84bbd-2ad3-4d57-bcc6-09d6098e020e: !Template + answer_choices: A ||| B + id: 80f84bbd-2ad3-4d57-bcc6-09d6098e020e + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 2 (bad-good, answer: B)' + reference: '' + c7cf5971-9888-4ca2-9400-a2c0f3dd42fd: !Template + answer_choices: null + id: c7cf5971-9888-4ca2-9400-a2c0f3dd42fd + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2 + reference: '' + ce043cec-56e6-4fd6-9332-60be62e5a22b: !Template + answer_choices: B ||| A + id: ce043cec-56e6-4fd6-9332-60be62e5a22b + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice 1 (good-bad, answer: A)' + reference: '' diff --git a/promptsource/templates/blimp/superlative_quantifiers_2/templates.yaml b/promptsource/templates/blimp/superlative_quantifiers_2/templates.yaml new file mode 100644 index 000000000..29c22e6d0 --- /dev/null +++ b/promptsource/templates/blimp/superlative_quantifiers_2/templates.yaml @@ -0,0 +1,235 @@ +dataset: blimp +subset: superlative_quantifiers_2 +templates: + 199def9a-fb9e-4594-8eae-e188ae4beb4a: !Template + answer_choices: null + id: 199def9a-fb9e-4594-8eae-e188ae4beb4a + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B_2 + reference: '' + 5165fa6b-b270-4b27-8394-3a6b9712854f: !Template + answer_choices: Yes ||| No + id: 5165fa6b-b270-4b27-8394-3a6b9712854f + jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' + 64300c9d-758c-4091-8977-e9f7e73c899c: !Template + answer_choices: null + id: 64300c9d-758c-4091-8977-e9f7e73c899c + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2 + reference: '' + 8ecc59b8-a388-47ad-b1e3-bd1eea549b9c: !Template + answer_choices: Yes ||| No + id: 8ecc59b8-a388-47ad-b1e3-bd1eea549b9c + jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' + 9309abba-e242-4512-9cee-13e77c6e4753: !Template + answer_choices: B ||| A + id: 9309abba-e242-4512-9cee-13e77c6e4753 + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice, 2 (bad-good, answer: B)' + reference: '' + 945bec39-e0d9-415e-9808-ff014239515a: !Template + answer_choices: A ||| B + id: 945bec39-e0d9-415e-9808-ff014239515a + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 1 (good-bad, answer: A)' + reference: '' + a8505fda-81c6-4f24-9b81-21d37241cdf2: !Template + answer_choices: null + id: a8505fda-81c6-4f24-9b81-21d37241cdf2 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B + reference: '' + b09e0d35-67e7-42b1-8af1-7328a3ebee15: !Template + answer_choices: A ||| B + id: b09e0d35-67e7-42b1-8af1-7328a3ebee15 + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 2 (bad-good, answer: B)' + reference: '' + bd488f67-ebf7-4111-ac11-479db18eff7e: !Template + answer_choices: B ||| A + id: bd488f67-ebf7-4111-ac11-479db18eff7e + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice 1 (good-bad, answer: A)' + reference: '' + cb3e52b0-ce14-4a7a-a6e5-ca0bd81de825: !Template + answer_choices: null + id: cb3e52b0-ce14-4a7a-a6e5-ca0bd81de825 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2_2 + reference: '' diff --git a/promptsource/templates/blimp/tough_vs_raising_1/templates.yaml b/promptsource/templates/blimp/tough_vs_raising_1/templates.yaml new file mode 100644 index 000000000..2097f58da --- /dev/null +++ b/promptsource/templates/blimp/tough_vs_raising_1/templates.yaml @@ -0,0 +1,235 @@ +dataset: blimp +subset: tough_vs_raising_1 +templates: + 3ee110da-2c50-48c1-9b06-c074744d4c76: !Template + answer_choices: Yes ||| No + id: 3ee110da-2c50-48c1-9b06-c074744d4c76 + jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' + 61438af9-e8fd-43b5-8248-518fb7cf22eb: !Template + answer_choices: A ||| B + id: 61438af9-e8fd-43b5-8248-518fb7cf22eb + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 1 (good-bad, answer: A)' + reference: '' + 70bb1415-b071-4402-a323-5d605590c5b4: !Template + answer_choices: A ||| B + id: 70bb1415-b071-4402-a323-5d605590c5b4 + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 2 (bad-good, answer: B)' + reference: '' + 7e30fe59-f359-434f-8697-e90c8afd892e: !Template + answer_choices: null + id: 7e30fe59-f359-434f-8697-e90c8afd892e + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B_2 + reference: '' + 7fb521b7-26ca-4f44-bce7-e3978960281b: !Template + answer_choices: Yes ||| No + id: 7fb521b7-26ca-4f44-bce7-e3978960281b + jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' + 860e2367-ebda-41db-8225-a261f829438e: !Template + answer_choices: null + id: 860e2367-ebda-41db-8225-a261f829438e + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B + reference: '' + 861bf826-2d14-4e1c-946c-28f6927bd7c9: !Template + answer_choices: B ||| A + id: 861bf826-2d14-4e1c-946c-28f6927bd7c9 + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice 1 (good-bad, answer: A)' + reference: '' + 8c774216-21ea-49cd-9256-3bd0f607f9cc: !Template + answer_choices: B ||| A + id: 8c774216-21ea-49cd-9256-3bd0f607f9cc + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice, 2 (bad-good, answer: B)' + reference: '' + b1562d73-9f9c-42df-ad17-f71b412d86e7: !Template + answer_choices: null + id: b1562d73-9f9c-42df-ad17-f71b412d86e7 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2_2 + reference: '' + f687b5b2-cc13-46a0-884a-d3db1d7aa515: !Template + answer_choices: null + id: f687b5b2-cc13-46a0-884a-d3db1d7aa515 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2 + reference: '' diff --git a/promptsource/templates/blimp/tough_vs_raising_2/templates.yaml b/promptsource/templates/blimp/tough_vs_raising_2/templates.yaml new file mode 100644 index 000000000..8a4943201 --- /dev/null +++ b/promptsource/templates/blimp/tough_vs_raising_2/templates.yaml @@ -0,0 +1,235 @@ +dataset: blimp +subset: tough_vs_raising_2 +templates: + 125ed492-f4df-4621-908f-7b614eaaf1e1: !Template + answer_choices: null + id: 125ed492-f4df-4621-908f-7b614eaaf1e1 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B_2 + reference: '' + 1f82e12a-2abc-4666-b671-b632d0865fb1: !Template + answer_choices: Yes ||| No + id: 1f82e12a-2abc-4666-b671-b632d0865fb1 + jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' + 437d29bb-252c-49a8-9aa4-85dc5079a195: !Template + answer_choices: B ||| A + id: 437d29bb-252c-49a8-9aa4-85dc5079a195 + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice, 2 (bad-good, answer: B)' + reference: '' + 49c1e0e8-e1b3-4f83-b0f1-21aabecfc149: !Template + answer_choices: B ||| A + id: 49c1e0e8-e1b3-4f83-b0f1-21aabecfc149 + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice 1 (good-bad, answer: A)' + reference: '' + 5c4cb780-a52a-4e9a-b654-5ea6ec7e05e4: !Template + answer_choices: Yes ||| No + id: 5c4cb780-a52a-4e9a-b654-5ea6ec7e05e4 + jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' + 65fac625-38ad-4b82-9532-332cb140ee3b: !Template + answer_choices: null + id: 65fac625-38ad-4b82-9532-332cb140ee3b + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2_2 + reference: '' + 8e41bb9d-9136-415f-907c-f5a84c043341: !Template + answer_choices: null + id: 8e41bb9d-9136-415f-907c-f5a84c043341 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2 + reference: '' + b97f4ee5-62df-4f8c-9a8a-603ba3d226f7: !Template + answer_choices: null + id: b97f4ee5-62df-4f8c-9a8a-603ba3d226f7 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B + reference: '' + cc816098-02f5-4071-8b01-ebbc34910f0d: !Template + answer_choices: A ||| B + id: cc816098-02f5-4071-8b01-ebbc34910f0d + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 1 (good-bad, answer: A)' + reference: '' + fdf5e28e-3f15-45a1-8d88-942a4bfac780: !Template + answer_choices: A ||| B + id: fdf5e28e-3f15-45a1-8d88-942a4bfac780 + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 2 (bad-good, answer: B)' + reference: '' diff --git a/promptsource/templates/blimp/transitive/templates.yaml b/promptsource/templates/blimp/transitive/templates.yaml new file mode 100644 index 000000000..39acd17a1 --- /dev/null +++ b/promptsource/templates/blimp/transitive/templates.yaml @@ -0,0 +1,235 @@ +dataset: blimp +subset: transitive +templates: + 11ba8b9d-b4c8-45b7-8e47-323cd3e5f636: !Template + answer_choices: Yes ||| No + id: 11ba8b9d-b4c8-45b7-8e47-323cd3e5f636 + jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' + 2097f653-2bd7-48f5-9682-57da525636c8: !Template + answer_choices: B ||| A + id: 2097f653-2bd7-48f5-9682-57da525636c8 + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice 1 (good-bad, answer: A)' + reference: '' + 2a535f9f-2d23-4875-b458-48fddd66f721: !Template + answer_choices: Yes ||| No + id: 2a535f9f-2d23-4875-b458-48fddd66f721 + jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' + 2c399b26-3521-4290-83d0-5652e312b5f6: !Template + answer_choices: null + id: 2c399b26-3521-4290-83d0-5652e312b5f6 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2 + reference: '' + 4a0cdb8a-3e9f-4f9f-9184-bfc68f57f41e: !Template + answer_choices: B ||| A + id: 4a0cdb8a-3e9f-4f9f-9184-bfc68f57f41e + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice, 2 (bad-good, answer: B)' + reference: '' + 674cdc46-5951-4e84-b44a-701e9d1661a0: !Template + answer_choices: null + id: 674cdc46-5951-4e84-b44a-701e9d1661a0 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2_2 + reference: '' + 9ed761fb-b0aa-4322-8b5d-85ad42904150: !Template + answer_choices: null + id: 9ed761fb-b0aa-4322-8b5d-85ad42904150 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B + reference: '' + c52ccf34-087a-4ffa-8281-55d839bf6dd0: !Template + answer_choices: A ||| B + id: c52ccf34-087a-4ffa-8281-55d839bf6dd0 + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 2 (bad-good, answer: B)' + reference: '' + d2783acf-666c-4e1f-9597-d23d092e357d: !Template + answer_choices: null + id: d2783acf-666c-4e1f-9597-d23d092e357d + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B_2 + reference: '' + f39bda9f-f80e-4933-a794-3b907d775b07: !Template + answer_choices: A ||| B + id: f39bda9f-f80e-4933-a794-3b907d775b07 + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 1 (good-bad, answer: A)' + reference: '' diff --git a/promptsource/templates/blimp/wh_island/templates.yaml b/promptsource/templates/blimp/wh_island/templates.yaml new file mode 100644 index 000000000..bd148ce34 --- /dev/null +++ b/promptsource/templates/blimp/wh_island/templates.yaml @@ -0,0 +1,235 @@ +dataset: blimp +subset: wh_island +templates: + 0478611e-bb77-4f95-84b8-bd9a23170584: !Template + answer_choices: A ||| B + id: 0478611e-bb77-4f95-84b8-bd9a23170584 + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 2 (bad-good, answer: B)' + reference: '' + 2f809829-d69e-4e3f-bafb-dc14c24d62b1: !Template + answer_choices: A ||| B + id: 2f809829-d69e-4e3f-bafb-dc14c24d62b1 + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 1 (good-bad, answer: A)' + reference: '' + 3f141165-53f6-4e23-a14d-3c1d0e9f07ff: !Template + answer_choices: Yes ||| No + id: 3f141165-53f6-4e23-a14d-3c1d0e9f07ff + jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' + 785b8d5a-1dda-4c2a-b820-36f6c937965f: !Template + answer_choices: null + id: 785b8d5a-1dda-4c2a-b820-36f6c937965f + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2_2 + reference: '' + 9731dd38-a7b5-4d71-be48-9b550db62440: !Template + answer_choices: B ||| A + id: 9731dd38-a7b5-4d71-be48-9b550db62440 + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice 1 (good-bad, answer: A)' + reference: '' + b15bad58-7860-44e3-8158-9c7aa13553da: !Template + answer_choices: Yes ||| No + id: b15bad58-7860-44e3-8158-9c7aa13553da + jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' + d78dc0a5-83b2-4e0b-b553-863fe4f06f4c: !Template + answer_choices: null + id: d78dc0a5-83b2-4e0b-b553-863fe4f06f4c + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B + reference: '' + e70949d2-9b57-4164-8cab-94a342aecca2: !Template + answer_choices: B ||| A + id: e70949d2-9b57-4164-8cab-94a342aecca2 + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice, 2 (bad-good, answer: B)' + reference: '' + f1072d69-55eb-435c-8458-8ed626da0d69: !Template + answer_choices: null + id: f1072d69-55eb-435c-8458-8ed626da0d69 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2 + reference: '' + f1445c4c-fb5d-41a2-b618-be2754c32bba: !Template + answer_choices: null + id: f1445c4c-fb5d-41a2-b618-be2754c32bba + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B_2 + reference: '' diff --git a/promptsource/templates/blimp/wh_questions_object_gap/templates.yaml b/promptsource/templates/blimp/wh_questions_object_gap/templates.yaml new file mode 100644 index 000000000..3558630f7 --- /dev/null +++ b/promptsource/templates/blimp/wh_questions_object_gap/templates.yaml @@ -0,0 +1,235 @@ +dataset: blimp +subset: wh_questions_object_gap +templates: + 03396fdd-6d6e-4324-a2e8-19acf2208d1b: !Template + answer_choices: Yes ||| No + id: 03396fdd-6d6e-4324-a2e8-19acf2208d1b + jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' + 1238b84f-7716-4416-a519-7338678392e0: !Template + answer_choices: B ||| A + id: 1238b84f-7716-4416-a519-7338678392e0 + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice, 2 (bad-good, answer: B)' + reference: '' + 46c0813a-7349-49a2-920e-9e1e95436950: !Template + answer_choices: B ||| A + id: 46c0813a-7349-49a2-920e-9e1e95436950 + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice 1 (good-bad, answer: A)' + reference: '' + 63f14809-6c03-43be-bcb0-69612e40a79e: !Template + answer_choices: null + id: 63f14809-6c03-43be-bcb0-69612e40a79e + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2_2 + reference: '' + 66b8a7a0-2b7d-49c6-8f8e-56f7176c4d5d: !Template + answer_choices: null + id: 66b8a7a0-2b7d-49c6-8f8e-56f7176c4d5d + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B_2 + reference: '' + ca560889-5e9c-4b01-a77f-2c21e3720e05: !Template + answer_choices: Yes ||| No + id: ca560889-5e9c-4b01-a77f-2c21e3720e05 + jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' + cbd4a91a-afd5-4c2b-b77e-3b89c48651bc: !Template + answer_choices: null + id: cbd4a91a-afd5-4c2b-b77e-3b89c48651bc + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B + reference: '' + d2c46389-8a3a-4df9-aeee-2446d6a8cd90: !Template + answer_choices: A ||| B + id: d2c46389-8a3a-4df9-aeee-2446d6a8cd90 + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 2 (bad-good, answer: B)' + reference: '' + d864211d-ec04-44ac-a00d-04291ec4298c: !Template + answer_choices: null + id: d864211d-ec04-44ac-a00d-04291ec4298c + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2 + reference: '' + d9f30b1b-df3f-40c6-9268-d5980b971c21: !Template + answer_choices: A ||| B + id: d9f30b1b-df3f-40c6-9268-d5980b971c21 + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 1 (good-bad, answer: A)' + reference: '' diff --git a/promptsource/templates/blimp/wh_questions_subject_gap/templates.yaml b/promptsource/templates/blimp/wh_questions_subject_gap/templates.yaml new file mode 100644 index 000000000..749c20c41 --- /dev/null +++ b/promptsource/templates/blimp/wh_questions_subject_gap/templates.yaml @@ -0,0 +1,235 @@ +dataset: blimp +subset: wh_questions_subject_gap +templates: + 1f785875-b311-4dc2-9d21-ec53e25917fa: !Template + answer_choices: B ||| A + id: 1f785875-b311-4dc2-9d21-ec53e25917fa + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice 1 (good-bad, answer: A)' + reference: '' + 2e0a281d-6001-48a9-a2f3-8862da48c2a7: !Template + answer_choices: null + id: 2e0a281d-6001-48a9-a2f3-8862da48c2a7 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2_2 + reference: '' + 4ad52c69-004f-44f5-9549-e8361c05f517: !Template + answer_choices: A ||| B + id: 4ad52c69-004f-44f5-9549-e8361c05f517 + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 1 (good-bad, answer: A)' + reference: '' + 79a7017d-589a-4680-bd49-5ae6584434f5: !Template + answer_choices: Yes ||| No + id: 79a7017d-589a-4680-bd49-5ae6584434f5 + jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' + 79e1d06a-9b7d-41b9-b7bc-5d77476b718b: !Template + answer_choices: A ||| B + id: 79e1d06a-9b7d-41b9-b7bc-5d77476b718b + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 2 (bad-good, answer: B)' + reference: '' + 7d72eb40-e6ca-41f8-8850-cdfd13ad04d5: !Template + answer_choices: null + id: 7d72eb40-e6ca-41f8-8850-cdfd13ad04d5 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B + reference: '' + 7e7a2b14-e49b-4559-8f97-dc0867d65a1a: !Template + answer_choices: Yes ||| No + id: 7e7a2b14-e49b-4559-8f97-dc0867d65a1a + jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' + 868aa4e2-2179-4f75-8ec2-b0499929536c: !Template + answer_choices: null + id: 868aa4e2-2179-4f75-8ec2-b0499929536c + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2 + reference: '' + ac68eb62-ac1d-4697-9ded-b7bd76f868d7: !Template + answer_choices: null + id: ac68eb62-ac1d-4697-9ded-b7bd76f868d7 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B_2 + reference: '' + ebff6ddd-90c7-42b5-adbe-d38cb9f6697d: !Template + answer_choices: B ||| A + id: ebff6ddd-90c7-42b5-adbe-d38cb9f6697d + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice, 2 (bad-good, answer: B)' + reference: '' diff --git a/promptsource/templates/blimp/wh_questions_subject_gap_long_distance/templates.yaml b/promptsource/templates/blimp/wh_questions_subject_gap_long_distance/templates.yaml new file mode 100644 index 000000000..0111eb941 --- /dev/null +++ b/promptsource/templates/blimp/wh_questions_subject_gap_long_distance/templates.yaml @@ -0,0 +1,235 @@ +dataset: blimp +subset: wh_questions_subject_gap_long_distance +templates: + 013d9cee-cce7-4552-9b4f-464aede20959: !Template + answer_choices: B ||| A + id: 013d9cee-cce7-4552-9b4f-464aede20959 + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice, 2 (bad-good, answer: B)' + reference: '' + 2948a7ac-5d31-4f2d-8458-78247861bd2a: !Template + answer_choices: Yes ||| No + id: 2948a7ac-5d31-4f2d-8458-78247861bd2a + jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' + 33e31879-6ab5-412a-aeb2-a77e5f61ef34: !Template + answer_choices: B ||| A + id: 33e31879-6ab5-412a-aeb2-a77e5f61ef34 + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice 1 (good-bad, answer: A)' + reference: '' + 6ecdd0a7-f8b5-42b8-b011-b4b5221a6082: !Template + answer_choices: A ||| B + id: 6ecdd0a7-f8b5-42b8-b011-b4b5221a6082 + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 2 (bad-good, answer: B)' + reference: '' + 9691d735-101f-4753-b9c8-269d8197a443: !Template + answer_choices: null + id: 9691d735-101f-4753-b9c8-269d8197a443 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2 + reference: '' + 9e095ece-29ed-448c-8f1b-2796ce04bced: !Template + answer_choices: null + id: 9e095ece-29ed-448c-8f1b-2796ce04bced + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B_2 + reference: '' + bc82b6e1-c0ae-4b46-ad01-eb7eb9b84745: !Template + answer_choices: Yes ||| No + id: bc82b6e1-c0ae-4b46-ad01-eb7eb9b84745 + jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' + c2543c63-0831-4492-8bad-1833931c63d4: !Template + answer_choices: A ||| B + id: c2543c63-0831-4492-8bad-1833931c63d4 + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 1 (good-bad, answer: A)' + reference: '' + faf4abab-c36f-4750-9ad7-2e7d780c05b3: !Template + answer_choices: null + id: faf4abab-c36f-4750-9ad7-2e7d780c05b3 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B + reference: '' + fb6b40ec-bada-4feb-b17e-403ce4d60d9f: !Template + answer_choices: null + id: fb6b40ec-bada-4feb-b17e-403ce4d60d9f + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2_2 + reference: '' diff --git a/promptsource/templates/blimp/wh_vs_that_no_gap/templates.yaml b/promptsource/templates/blimp/wh_vs_that_no_gap/templates.yaml new file mode 100644 index 000000000..6afcbc645 --- /dev/null +++ b/promptsource/templates/blimp/wh_vs_that_no_gap/templates.yaml @@ -0,0 +1,235 @@ +dataset: blimp +subset: wh_vs_that_no_gap +templates: + 2048ce44-7631-4a28-aa1f-e246af495dd3: !Template + answer_choices: A ||| B + id: 2048ce44-7631-4a28-aa1f-e246af495dd3 + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 2 (bad-good, answer: B)' + reference: '' + 4ba7fc41-f60b-4a17-a4be-7f895de84a5a: !Template + answer_choices: null + id: 4ba7fc41-f60b-4a17-a4be-7f895de84a5a + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2 + reference: '' + 55d6d95e-1787-4fc7-b536-794afef94c81: !Template + answer_choices: null + id: 55d6d95e-1787-4fc7-b536-794afef94c81 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2_2 + reference: '' + 9cc4732b-7fa3-4bcc-bf18-d19113463db2: !Template + answer_choices: B ||| A + id: 9cc4732b-7fa3-4bcc-bf18-d19113463db2 + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice 1 (good-bad, answer: A)' + reference: '' + a0a33ec9-1c28-4b03-ba0c-3df3ceb64db1: !Template + answer_choices: A ||| B + id: a0a33ec9-1c28-4b03-ba0c-3df3ceb64db1 + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 1 (good-bad, answer: A)' + reference: '' + b0ec8021-bc27-45cb-80cd-54622f8f6eea: !Template + answer_choices: B ||| A + id: b0ec8021-bc27-45cb-80cd-54622f8f6eea + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice, 2 (bad-good, answer: B)' + reference: '' + c63932ed-7a24-416a-ad09-bcfee969322b: !Template + answer_choices: Yes ||| No + id: c63932ed-7a24-416a-ad09-bcfee969322b + jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' + da3309a0-3693-4cdb-be2b-2b6cb81b41e6: !Template + answer_choices: null + id: da3309a0-3693-4cdb-be2b-2b6cb81b41e6 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B + reference: '' + ef78f10f-aadf-4cc0-9c63-7222d63796c6: !Template + answer_choices: null + id: ef78f10f-aadf-4cc0-9c63-7222d63796c6 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B_2 + reference: '' + f7d89f75-6fcf-42be-a1a5-0ac49184f282: !Template + answer_choices: Yes ||| No + id: f7d89f75-6fcf-42be-a1a5-0ac49184f282 + jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' diff --git a/promptsource/templates/blimp/wh_vs_that_no_gap_long_distance/templates.yaml b/promptsource/templates/blimp/wh_vs_that_no_gap_long_distance/templates.yaml new file mode 100644 index 000000000..8820974dd --- /dev/null +++ b/promptsource/templates/blimp/wh_vs_that_no_gap_long_distance/templates.yaml @@ -0,0 +1,235 @@ +dataset: blimp +subset: wh_vs_that_no_gap_long_distance +templates: + 246ed2f2-f3e7-4a12-b0da-282e961a049f: !Template + answer_choices: A ||| B + id: 246ed2f2-f3e7-4a12-b0da-282e961a049f + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 2 (bad-good, answer: B)' + reference: '' + 373fb84b-81b1-4306-94a7-5052b4661061: !Template + answer_choices: null + id: 373fb84b-81b1-4306-94a7-5052b4661061 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2_2 + reference: '' + 4963c845-8818-4201-98a7-7752d67f0eb6: !Template + answer_choices: null + id: 4963c845-8818-4201-98a7-7752d67f0eb6 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2 + reference: '' + 7f680752-38cf-4ef6-91b1-9ce13f2ff6e2: !Template + answer_choices: B ||| A + id: 7f680752-38cf-4ef6-91b1-9ce13f2ff6e2 + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice, 2 (bad-good, answer: B)' + reference: '' + c044e8d3-cb02-40ad-a34c-93fa78c99dda: !Template + answer_choices: B ||| A + id: c044e8d3-cb02-40ad-a34c-93fa78c99dda + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice 1 (good-bad, answer: A)' + reference: '' + d6599adf-d39a-497f-abc1-a19a718dc5b5: !Template + answer_choices: Yes ||| No + id: d6599adf-d39a-497f-abc1-a19a718dc5b5 + jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' + d79f19df-ff07-44f5-ae88-6d89322da1d9: !Template + answer_choices: A ||| B + id: d79f19df-ff07-44f5-ae88-6d89322da1d9 + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 1 (good-bad, answer: A)' + reference: '' + ddffc67a-181e-4716-a465-17ab9db85e22: !Template + answer_choices: null + id: ddffc67a-181e-4716-a465-17ab9db85e22 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B_2 + reference: '' + f0a06b70-e486-4d97-a378-1c9984ce7e00: !Template + answer_choices: null + id: f0a06b70-e486-4d97-a378-1c9984ce7e00 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B + reference: '' + f61f57ef-8bba-4be1-8e3b-4148239ca7d5: !Template + answer_choices: Yes ||| No + id: f61f57ef-8bba-4be1-8e3b-4148239ca7d5 + jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' diff --git a/promptsource/templates/blimp/wh_vs_that_with_gap/templates.yaml b/promptsource/templates/blimp/wh_vs_that_with_gap/templates.yaml new file mode 100644 index 000000000..c78e00d86 --- /dev/null +++ b/promptsource/templates/blimp/wh_vs_that_with_gap/templates.yaml @@ -0,0 +1,235 @@ +dataset: blimp +subset: wh_vs_that_with_gap +templates: + 017b9ef0-83b1-46e6-ab86-747e9f242d28: !Template + answer_choices: null + id: 017b9ef0-83b1-46e6-ab86-747e9f242d28 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2 + reference: '' + 3bce33c7-3fd7-4642-a5fd-f03ea801fd48: !Template + answer_choices: Yes ||| No + id: 3bce33c7-3fd7-4642-a5fd-f03ea801fd48 + jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' + 3d7ea5ae-840e-4778-a53b-79ea82d90648: !Template + answer_choices: A ||| B + id: 3d7ea5ae-840e-4778-a53b-79ea82d90648 + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 2 (bad-good, answer: B)' + reference: '' + 551e7c19-5c94-4ec8-b66d-751b37314489: !Template + answer_choices: null + id: 551e7c19-5c94-4ec8-b66d-751b37314489 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B + reference: '' + 57cd4af1-97ab-49b2-8be6-771272cd2793: !Template + answer_choices: Yes ||| No + id: 57cd4af1-97ab-49b2-8be6-771272cd2793 + jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' + 6675a3b5-3e1b-491f-9b36-cdec1d162230: !Template + answer_choices: A ||| B + id: 6675a3b5-3e1b-491f-9b36-cdec1d162230 + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 1 (good-bad, answer: A)' + reference: '' + 6d3ae668-08f6-49b8-ade8-8dab0a52af75: !Template + answer_choices: null + id: 6d3ae668-08f6-49b8-ade8-8dab0a52af75 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B_2 + reference: '' + af9ec49c-76d8-41ec-86d3-c855726de0e4: !Template + answer_choices: B ||| A + id: af9ec49c-76d8-41ec-86d3-c855726de0e4 + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice, 2 (bad-good, answer: B)' + reference: '' + cfc1dddb-d794-4a0e-921f-085c9fb1d8db: !Template + answer_choices: B ||| A + id: cfc1dddb-d794-4a0e-921f-085c9fb1d8db + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice 1 (good-bad, answer: A)' + reference: '' + d0a0cbd1-cb36-4b0e-988b-655bd8fa1209: !Template + answer_choices: null + id: d0a0cbd1-cb36-4b0e-988b-655bd8fa1209 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2_2 + reference: '' diff --git a/promptsource/templates/blimp/wh_vs_that_with_gap_long_distance/templates.yaml b/promptsource/templates/blimp/wh_vs_that_with_gap_long_distance/templates.yaml new file mode 100644 index 000000000..0794fb82e --- /dev/null +++ b/promptsource/templates/blimp/wh_vs_that_with_gap_long_distance/templates.yaml @@ -0,0 +1,235 @@ +dataset: blimp +subset: wh_vs_that_with_gap_long_distance +templates: + 1bf4224c-3cd2-4df1-ab50-f966ffa2a7a4: !Template + answer_choices: null + id: 1bf4224c-3cd2-4df1-ab50-f966ffa2a7a4 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B + reference: '' + 22bbdd4a-1ad9-42b9-a4e0-33c346bd3fef: !Template + answer_choices: B ||| A + id: 22bbdd4a-1ad9-42b9-a4e0-33c346bd3fef + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice 1 (good-bad, answer: A)' + reference: '' + 4c34df87-6349-485c-95bf-b62faaec20c8: !Template + answer_choices: null + id: 4c34df87-6349-485c-95bf-b62faaec20c8 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2_2 + reference: '' + b0a642c6-99a5-4317-bbbd-9fef33b8f613: !Template + answer_choices: null + id: b0a642c6-99a5-4317-bbbd-9fef33b8f613 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B_2 + reference: '' + be12b2b2-8884-448e-90ed-1df65269119b: !Template + answer_choices: null + id: be12b2b2-8884-448e-90ed-1df65269119b + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2 + reference: '' + d1356cc1-f721-4d33-be47-9355043c0fe0: !Template + answer_choices: A ||| B + id: d1356cc1-f721-4d33-be47-9355043c0fe0 + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 2 (bad-good, answer: B)' + reference: '' + ddd157be-6df7-4be8-9864-3638ec9eae93: !Template + answer_choices: Yes ||| No + id: ddd157be-6df7-4be8-9864-3638ec9eae93 + jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' + e0b75387-7860-433c-a08b-02fd87172c60: !Template + answer_choices: A ||| B + id: e0b75387-7860-433c-a08b-02fd87172c60 + jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ + sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice 1 (good-bad, answer: A)' + reference: '' + ea8e839d-6fe0-4a37-9f9a-83495102a67e: !Template + answer_choices: B ||| A + id: ea8e839d-6fe0-4a37-9f9a-83495102a67e + jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good + }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'B/A choice, 2 (bad-good, answer: B)' + reference: '' + fe404484-5c73-4187-acb7-c7e79c2da2e4: !Template + answer_choices: Yes ||| No + id: fe404484-5c73-4187-acb7-c7e79c2da2e4 + jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] + }} + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' From 30a1cda49124d7e979b2e11420a304a24a5c4689 Mon Sep 17 00:00:00 2001 From: cookie Date: Tue, 26 Apr 2022 11:17:26 -0400 Subject: [PATCH 10/16] Switched to randomized prompts. --- .../blimp/adjunct_island/templates.yaml | 134 ++++++--- .../anaphor_gender_agreement/templates.yaml | 212 +++++++++----- .../anaphor_number_agreement/templates.yaml | 246 +++++++++++------ .../animate_subject_passive/templates.yaml | 220 +++++++++------ .../animate_subject_trans/templates.yaml | 244 +++++++++++------ .../templates/blimp/causative/templates.yaml | 254 ++++++++++------- .../blimp/complex_NP_island/templates.yaml | 224 +++++++++------ .../templates.yaml | 200 +++++++++----- .../templates.yaml | 220 +++++++++------ .../templates.yaml | 222 +++++++++------ .../templates.yaml | 258 +++++++++++------- .../templates.yaml | 222 +++++++++------ .../templates.yaml | 230 ++++++++++------ .../templates.yaml | 220 +++++++++------ .../templates.yaml | 206 +++++++++----- .../templates.yaml | 214 ++++++++++----- .../templates.yaml | 236 ++++++++++------ .../templates.yaml | 242 ++++++++++------ .../templates.yaml | 236 ++++++++++------ .../blimp/drop_argument/templates.yaml | 214 ++++++++++----- .../blimp/ellipsis_n_bar_1/templates.yaml | 232 ++++++++++------ .../blimp/ellipsis_n_bar_2/templates.yaml | 238 ++++++++++------ .../templates.yaml | 234 ++++++++++------ .../templates.yaml | 228 ++++++++++------ .../templates.yaml | 204 +++++++++----- .../templates.yaml | 244 +++++++++++------ .../templates.yaml | 218 +++++++++------ .../templates/blimp/inchoative/templates.yaml | 226 +++++++++------ .../blimp/intransitive/templates.yaml | 234 ++++++++++------ .../templates.yaml | 238 ++++++++++------ .../templates.yaml | 224 +++++++++------ .../templates.yaml | 206 +++++++++----- .../templates.yaml | 222 +++++++++------ .../templates.yaml | 198 +++++++++----- .../templates.yaml | 218 +++++++++------ .../templates.yaml | 214 ++++++++++----- .../blimp/npi_present_1/templates.yaml | 204 +++++++++----- .../blimp/npi_present_2/templates.yaml | 250 ++++++++++------- .../only_npi_licensor_present/templates.yaml | 252 ++++++++++------- .../blimp/only_npi_scope/templates.yaml | 218 +++++++++------ .../templates/blimp/passive_1/templates.yaml | 216 +++++++++------ .../templates/blimp/passive_2/templates.yaml | 236 ++++++++++------ .../principle_A_c_command/templates.yaml | 222 +++++++++------ .../blimp/principle_A_case_1/templates.yaml | 232 ++++++++++------ .../blimp/principle_A_case_2/templates.yaml | 244 +++++++++++------ .../blimp/principle_A_domain_1/templates.yaml | 236 ++++++++++------ .../blimp/principle_A_domain_2/templates.yaml | 226 +++++++++------ .../blimp/principle_A_domain_3/templates.yaml | 242 ++++++++++------ .../principle_A_reconstruction/templates.yaml | 244 +++++++++++------ .../templates.yaml | 218 +++++++++------ .../templates.yaml | 210 +++++++++----- .../templates.yaml | 210 +++++++++----- .../templates.yaml | 244 +++++++++++------ .../sentential_subject_island/templates.yaml | 218 +++++++++------ .../superlative_quantifiers_1/templates.yaml | 220 +++++++++------ .../superlative_quantifiers_2/templates.yaml | 220 +++++++++------ .../blimp/tough_vs_raising_1/templates.yaml | 250 ++++++++++------- .../blimp/tough_vs_raising_2/templates.yaml | 236 ++++++++++------ .../templates/blimp/transitive/templates.yaml | 240 ++++++++++------ .../templates/blimp/wh_island/templates.yaml | 218 +++++++++------ .../wh_questions_object_gap/templates.yaml | 238 ++++++++++------ .../wh_questions_subject_gap/templates.yaml | 220 +++++++++------ .../templates.yaml | 230 ++++++++++------ .../blimp/wh_vs_that_no_gap/templates.yaml | 222 +++++++++------ .../templates.yaml | 218 +++++++++------ .../blimp/wh_vs_that_with_gap/templates.yaml | 210 +++++++++----- .../templates.yaml | 212 +++++++++----- 67 files changed, 9688 insertions(+), 5400 deletions(-) diff --git a/promptsource/templates/blimp/adjunct_island/templates.yaml b/promptsource/templates/blimp/adjunct_island/templates.yaml index e3b77921b..1a56a5ce8 100644 --- a/promptsource/templates/blimp/adjunct_island/templates.yaml +++ b/promptsource/templates/blimp/adjunct_island/templates.yaml @@ -4,28 +4,40 @@ templates: 3d2821a7-1c9c-44dd-ad23-e964b0998d1b: !Template answer_choices: A ||| B id: 3d2821a7-1c9c-44dd-ad23-e964b0998d1b - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'A/B choice 2 (bad-good, answer: B)' - reference: '' - 41594c98-baea-4b1f-a631-27842cd363fd: !Template - answer_choices: B ||| A - id: 41594c98-baea-4b1f-a631-27842cd363fd - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] - }}' + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'B/A choice 1 (good-bad, answer: A)' + name: A/B choice randomized (choice order reversed) reference: '' 6e71cd4b-882c-4458-9cd4-fd83f849ad09: !Template answer_choices: null @@ -105,26 +117,33 @@ templates: original_task: true name: random_choice_1_2 reference: '' - 88abc121-43ea-4b67-92e3-d762884d391d: !Template - answer_choices: B ||| A - id: 88abc121-43ea-4b67-92e3-d762884d391d - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] - }}' + 8bb3b001-cda4-43a3-8692-f191393bb2ef: !Template + answer_choices: Yes ||| No + id: 8bb3b001-cda4-43a3-8692-f191393bb2ef + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'B/A choice, 2 (bad-good, answer: B)' + original_task: false + name: Yes/No binary judgment for bad sentences (choice order reversed) reference: '' 8f354d74-5ffb-4cbc-b119-b79f2db00df3: !Template answer_choices: Yes ||| No id: 8f354d74-5ffb-4cbc-b119-b79f2db00df3 - jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] - }} + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata - choices_in_prompt: false + choices_in_prompt: true metrics: - Accuracy original_task: false @@ -172,10 +191,14 @@ templates: a1997343-0a20-4f8c-8939-a512f0f68485: !Template answer_choices: Yes ||| No id: a1997343-0a20-4f8c-8939-a512f0f68485 - jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] - }} + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata - choices_in_prompt: false + choices_in_prompt: true metrics: - Accuracy original_task: false @@ -184,15 +207,40 @@ templates: c45d5bac-968e-4e39-96b7-d18a5ddb225c: !Template answer_choices: A ||| B id: c45d5bac-968e-4e39-96b7-d18a5ddb225c - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] - }}' + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'A/B choice 1 (good-bad, answer: A)' + name: A/B choice randomized reference: '' d11a37d7-7c42-4ee0-96c5-ffd24ad7073e: !Template answer_choices: null @@ -233,3 +281,19 @@ templates: original_task: true name: random_choice_A_B reference: '' + f4f94eda-a7ac-485a-82d1-e36012771762: !Template + answer_choices: Yes ||| No + id: f4f94eda-a7ac-485a-82d1-e36012771762 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences (choice order reversed) + reference: '' diff --git a/promptsource/templates/blimp/anaphor_gender_agreement/templates.yaml b/promptsource/templates/blimp/anaphor_gender_agreement/templates.yaml index c4c6c4b00..6fb689db0 100644 --- a/promptsource/templates/blimp/anaphor_gender_agreement/templates.yaml +++ b/promptsource/templates/blimp/anaphor_gender_agreement/templates.yaml @@ -1,36 +1,35 @@ dataset: blimp subset: anaphor_gender_agreement templates: - 1a9f6ffa-2f8a-497f-8027-83247e8a3db7: !Template - answer_choices: null - id: 1a9f6ffa-2f8a-497f-8027-83247e8a3db7 + 10c299d4-88b2-4781-8202-accc479e5bf2: !Template + answer_choices: A ||| B + id: 10c299d4-88b2-4781-8202-accc479e5bf2 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -38,23 +37,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: A/B choice randomized (choice order reversed) reference: '' - 26edb4c7-2248-4f0e-af34-a4316bd7bb15: !Template - answer_choices: Yes ||| No - id: 26edb4c7-2248-4f0e-af34-a4316bd7bb15 - jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] - }} - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences - reference: '' - 440c3a37-3df7-456e-8f8f-68953ce605e8: !Template + 31007ad7-9fcc-431d-958b-55a9e3e85e62: !Template answer_choices: null - id: 440c3a37-3df7-456e-8f8f-68953ce605e8 + id: 31007ad7-9fcc-431d-958b-55a9e3e85e62 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer A or B. @@ -91,9 +78,47 @@ templates: original_task: true name: random_choice_A_B reference: '' - 4c2f4b2f-542e-4d04-8a5c-9a1fedfc4079: !Template + 34f21662-aac6-4dc6-9bfb-7a0975cfc9c8: !Template + answer_choices: A ||| B + id: 34f21662-aac6-4dc6-9bfb-7a0975cfc9c8 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: A/B choice randomized + reference: '' + 47617b60-4498-4b76-bc4f-590e0cf9fddb: !Template answer_choices: null - id: 4c2f4b2f-542e-4d04-8a5c-9a1fedfc4079 + id: 47617b60-4498-4b76-bc4f-590e0cf9fddb jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer A or B. @@ -130,48 +155,41 @@ templates: original_task: true name: random_choice_A_B_2 reference: '' - 5fc6b5cf-e083-485b-b727-7b1daf182102: !Template - answer_choices: A ||| B - id: 5fc6b5cf-e083-485b-b727-7b1daf182102 - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'A/B choice 2 (bad-good, answer: B)' - reference: '' - 6de15abd-3a32-42b0-a2e5-997487ef9f81: !Template - answer_choices: A ||| B - id: 6de15abd-3a32-42b0-a2e5-997487ef9f81 - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] - }}' + 5df5fcdc-81c8-4d6d-8b7e-57a0ea71d83c: !Template + answer_choices: Yes ||| No + id: 5df5fcdc-81c8-4d6d-8b7e-57a0ea71d83c + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'A/B choice 1 (good-bad, answer: A)' + original_task: false + name: Yes/No binary judgment for good sentences reference: '' - 710fe183-95cc-4228-b3c5-a141f0f56da1: !Template - answer_choices: B ||| A - id: 710fe183-95cc-4228-b3c5-a141f0f56da1 - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] - }}' + 6a81d250-ca60-4bd7-ad38-60f42a3e68bf: !Template + answer_choices: Yes ||| No + id: 6a81d250-ca60-4bd7-ad38-60f42a3e68bf + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'B/A choice, 2 (bad-good, answer: B)' + original_task: false + name: Yes/No binary judgment for good sentences (choice order reversed) reference: '' - 9fa6e2d6-430d-4f5e-b54f-7e1c237b2557: !Template + 6ddd2ea0-be02-4704-ab24-568cc4169776: !Template answer_choices: null - id: 9fa6e2d6-430d-4f5e-b54f-7e1c237b2557 + id: 6ddd2ea0-be02-4704-ab24-568cc4169776 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer 1 or 2. @@ -208,28 +226,74 @@ templates: original_task: true name: random_choice_1_2 reference: '' - af619299-690d-41a3-96da-c2369d152dbf: !Template - answer_choices: B ||| A - id: af619299-690d-41a3-96da-c2369d152dbf - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] - }}' + 710e6025-416c-4118-a2e0-d732ed9559a4: !Template + answer_choices: Yes ||| No + id: 710e6025-416c-4118-a2e0-d732ed9559a4 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'B/A choice 1 (good-bad, answer: A)' + original_task: false + name: Yes/No binary judgment for bad sentences (choice order reversed) reference: '' - e2f4b270-eac8-4649-9f34-1090186d2487: !Template + a5be9eb3-ce3c-4228-a764-6630b5d89944: !Template answer_choices: Yes ||| No - id: e2f4b270-eac8-4649-9f34-1090186d2487 - jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] - }} + id: a5be9eb3-ce3c-4228-a764-6630b5d89944 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata - choices_in_prompt: false + choices_in_prompt: true metrics: - Accuracy original_task: false name: Yes/No binary judgment for bad sentences reference: '' + efc454a6-899f-48a0-846c-de78fe3499da: !Template + answer_choices: null + id: efc454a6-899f-48a0-846c-de78fe3499da + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2_2 + reference: '' diff --git a/promptsource/templates/blimp/anaphor_number_agreement/templates.yaml b/promptsource/templates/blimp/anaphor_number_agreement/templates.yaml index a6b6aa006..c30fc2455 100644 --- a/promptsource/templates/blimp/anaphor_number_agreement/templates.yaml +++ b/promptsource/templates/blimp/anaphor_number_agreement/templates.yaml @@ -1,49 +1,51 @@ dataset: blimp subset: anaphor_number_agreement templates: - 15858e60-b678-4ae7-b5a4-98b792b95dc3: !Template - answer_choices: A ||| B - id: 15858e60-b678-4ae7-b5a4-98b792b95dc3 - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] - }}' + 0b65db41-53fc-4147-b1f3-e70a0095d57d: !Template + answer_choices: Yes ||| No + id: 0b65db41-53fc-4147-b1f3-e70a0095d57d + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'A/B choice 1 (good-bad, answer: A)' + original_task: false + name: Yes/No binary judgment for bad sentences (choice order reversed) reference: '' - 5396f9fa-2712-4af7-bdb0-68be1253c2d8: !Template - answer_choices: null - id: 5396f9fa-2712-4af7-bdb0-68be1253c2d8 + 384a6063-63cd-4fd1-85db-1d0e0b64f648: !Template + answer_choices: A ||| B + id: 384a6063-63cd-4fd1-85db-1d0e0b64f648 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} - {% if shuffled_order == 0 %} - - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -51,49 +53,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 - reference: '' - 7a8b9db6-969e-4e0a-8abd-b270cb414268: !Template - answer_choices: Yes ||| No - id: 7a8b9db6-969e-4e0a-8abd-b270cb414268 - jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] - }} - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences - reference: '' - 8ea37f01-e441-4b7b-9652-94ed6990f9ac: !Template - answer_choices: B ||| A - id: 8ea37f01-e441-4b7b-9652-94ed6990f9ac - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'B/A choice 1 (good-bad, answer: A)' - reference: '' - 993edeef-84eb-4c7d-92bc-c74187ea9054: !Template - answer_choices: A ||| B - id: 993edeef-84eb-4c7d-92bc-c74187ea9054 - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'A/B choice 2 (bad-good, answer: B)' + name: A/B choice randomized reference: '' - a0dc89be-5646-45de-a6bd-6c54a88be38e: !Template + 77503d37-b2e0-44db-b9a1-00c6f9cabed7: !Template answer_choices: null - id: a0dc89be-5646-45de-a6bd-6c54a88be38e + id: 77503d37-b2e0-44db-b9a1-00c6f9cabed7 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer A or B. @@ -130,34 +94,9 @@ templates: original_task: true name: random_choice_A_B reference: '' - c638fc37-0a6f-408f-bca6-98f6fa29c989: !Template - answer_choices: Yes ||| No - id: c638fc37-0a6f-408f-bca6-98f6fa29c989 - jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] - }} - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences - reference: '' - c8d6f39d-ad35-43ad-88bc-a2eb2757598c: !Template - answer_choices: B ||| A - id: c8d6f39d-ad35-43ad-88bc-a2eb2757598c - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'B/A choice, 2 (bad-good, answer: B)' - reference: '' - e6ab9771-3180-42bd-b374-d40a5ebb85c7: !Template + 7e77764d-bb84-4079-bcf4-e63e0a7909ea: !Template answer_choices: null - id: e6ab9771-3180-42bd-b374-d40a5ebb85c7 + id: 7e77764d-bb84-4079-bcf4-e63e0a7909ea jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -194,9 +133,9 @@ templates: original_task: true name: random_choice_1_2_2 reference: '' - f587556a-8f43-4ff6-975b-1c5d625edf6c: !Template + 82450d0b-0e57-4dd8-94d5-d79351b882dc: !Template answer_choices: null - id: f587556a-8f43-4ff6-975b-1c5d625edf6c + id: 82450d0b-0e57-4dd8-94d5-d79351b882dc jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer A or B. @@ -233,3 +172,128 @@ templates: original_task: true name: random_choice_A_B_2 reference: '' + 8c3bd71c-646b-440e-9f16-579f6d88af02: !Template + answer_choices: null + id: 8c3bd71c-646b-440e-9f16-579f6d88af02 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2 + reference: '' + ac349cca-b921-4dbe-8a01-38e924a924d0: !Template + answer_choices: A ||| B + id: ac349cca-b921-4dbe-8a01-38e924a924d0 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: A/B choice randomized (choice order reversed) + reference: '' + ca01676d-f0f8-42e0-9ab5-41031bfed413: !Template + answer_choices: Yes ||| No + id: ca01676d-f0f8-42e0-9ab5-41031bfed413 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' + cb1f9887-ee16-40c5-8421-3021fa8dd287: !Template + answer_choices: Yes ||| No + id: cb1f9887-ee16-40c5-8421-3021fa8dd287 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences (choice order reversed) + reference: '' + faa27b3f-2c13-4a30-bcf9-71a5a7a3f129: !Template + answer_choices: Yes ||| No + id: faa27b3f-2c13-4a30-bcf9-71a5a7a3f129 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' diff --git a/promptsource/templates/blimp/animate_subject_passive/templates.yaml b/promptsource/templates/blimp/animate_subject_passive/templates.yaml index a4a0f9da8..f003eaa40 100644 --- a/promptsource/templates/blimp/animate_subject_passive/templates.yaml +++ b/promptsource/templates/blimp/animate_subject_passive/templates.yaml @@ -1,50 +1,105 @@ dataset: blimp subset: animate_subject_passive templates: - 2aa95c35-d9c6-4bcb-bc3e-4e9add599cc3: !Template + 0c63ccdb-f4c4-48b7-8bff-9ef98e874c11: !Template answer_choices: A ||| B - id: 2aa95c35-d9c6-4bcb-bc3e-4e9add599cc3 - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] - }}' + id: 0c63ccdb-f4c4-48b7-8bff-9ef98e874c11 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'A/B choice 1 (good-bad, answer: A)' + name: A/B choice randomized (choice order reversed) reference: '' - 5a8a4633-d508-4f34-a4d5-8fc0e46a8ab4: !Template - answer_choices: B ||| A - id: 5a8a4633-d508-4f34-a4d5-8fc0e46a8ab4 - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] - }}' + 453cf076-9c83-474c-bc81-b8cba3668e85: !Template + answer_choices: null + id: 453cf076-9c83-474c-bc81-b8cba3668e85 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'B/A choice 1 (good-bad, answer: A)' + name: random_choice_1_2_2 reference: '' - 5c5ded69-0492-4495-8290-ada132a6161c: !Template + 47126c4b-302d-47e6-8605-f2765c678bb8: !Template answer_choices: Yes ||| No - id: 5c5ded69-0492-4495-8290-ada132a6161c - jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] - }} + id: 47126c4b-302d-47e6-8605-f2765c678bb8 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata - choices_in_prompt: false + choices_in_prompt: true metrics: - Accuracy original_task: false - name: Yes/No binary judgment for bad sentences + name: Yes/No binary judgment for good sentences (choice order reversed) reference: '' - a90f7a73-56b8-4bc0-bc7b-5ee760578101: !Template + 48567d36-352c-41bb-a25b-282ffae6c1e1: !Template answer_choices: null - id: a90f7a73-56b8-4bc0-bc7b-5ee760578101 + id: 48567d36-352c-41bb-a25b-282ffae6c1e1 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -76,50 +131,53 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: random_choice_1_2 reference: '' - bc68fae5-2635-4aef-9d7a-f931877f1c2d: !Template + b561a033-3b01-4aca-9ec4-9641e54a5fde: !Template answer_choices: Yes ||| No - id: bc68fae5-2635-4aef-9d7a-f931877f1c2d - jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] - }} + id: b561a033-3b01-4aca-9ec4-9641e54a5fde + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata - choices_in_prompt: false + choices_in_prompt: true metrics: - Accuracy original_task: false - name: Yes/No binary judgment for good sentences + name: Yes/No binary judgment for bad sentences (choice order reversed) reference: '' - bdb9186b-4c0e-41d2-9ba7-8f6cc824d90f: !Template - answer_choices: null - id: bdb9186b-4c0e-41d2-9ba7-8f6cc824d90f + b724ca94-7643-46c7-a64d-8a86b04fd48c: !Template + answer_choices: A ||| B + id: b724ca94-7643-46c7-a64d-8a86b04fd48c jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} - {% if shuffled_order == 0 %} - - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -127,26 +185,42 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: A/B choice randomized + reference: '' + cd2d6e9c-bc1b-4f43-8633-f2dc6f4845fc: !Template + answer_choices: Yes ||| No + id: cd2d6e9c-bc1b-4f43-8633-f2dc6f4845fc + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences reference: '' - c36492ef-9b69-4b4e-8332-1b5983fff753: !Template + cf18d556-acc0-4675-a28a-39deceff0cbc: !Template answer_choices: null - id: c36492ef-9b69-4b4e-8332-1b5983fff753 + id: cf18d556-acc0-4675-a28a-39deceff0cbc jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -154,11 +228,11 @@ templates: {% if shuffled_order == 0 %} - 1 + A {% else %} - 2 + B {% endif %}' metadata: !TemplateMetadata @@ -166,40 +240,30 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 - reference: '' - e6b8a2a5-f397-453c-83da-aa73d9906423: !Template - answer_choices: A ||| B - id: e6b8a2a5-f397-453c-83da-aa73d9906423 - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'A/B choice 2 (bad-good, answer: B)' + name: random_choice_A_B_2 reference: '' - e71bc916-ed97-4a14-8eb4-2871c7b5267a: !Template - answer_choices: B ||| A - id: e71bc916-ed97-4a14-8eb4-2871c7b5267a - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] - }}' + d9bdf45e-6574-4a1f-8bc6-16edc2c118a1: !Template + answer_choices: Yes ||| No + id: d9bdf45e-6574-4a1f-8bc6-16edc2c118a1 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'B/A choice, 2 (bad-good, answer: B)' + original_task: false + name: Yes/No binary judgment for good sentences reference: '' - e7d26950-94eb-4bf1-b7b2-958e9f7ca555: !Template + e55eb95e-29e1-4459-9782-58fba8b03ffc: !Template answer_choices: null - id: e7d26950-94eb-4bf1-b7b2-958e9f7ca555 + id: e55eb95e-29e1-4459-9782-58fba8b03ffc jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -231,5 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: random_choice_A_B reference: '' diff --git a/promptsource/templates/blimp/animate_subject_trans/templates.yaml b/promptsource/templates/blimp/animate_subject_trans/templates.yaml index 1b4b8132f..3dc0a4fbe 100644 --- a/promptsource/templates/blimp/animate_subject_trans/templates.yaml +++ b/promptsource/templates/blimp/animate_subject_trans/templates.yaml @@ -1,25 +1,12 @@ dataset: blimp subset: animate_subject_trans templates: - 2a836319-30fa-4ee4-b04c-2fdd21fabf8d: !Template - answer_choices: A ||| B - id: 2a836319-30fa-4ee4-b04c-2fdd21fabf8d - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'A/B choice 2 (bad-good, answer: B)' - reference: '' - 8eb6ce9f-6878-48e8-b210-8310fa11fb80: !Template + 185a0721-f8b5-49b8-9101-b97a70374247: !Template answer_choices: null - id: 8eb6ce9f-6878-48e8-b210-8310fa11fb80 + id: 185a0721-f8b5-49b8-9101-b97a70374247 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -51,14 +38,46 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: random_choice_1_2_2 reference: '' - a4b6ff3c-a6f9-49c6-9e5e-dee4d1505d25: !Template + 2835e45d-c778-468c-9ee7-bbd5dc8a7a92: !Template + answer_choices: Yes ||| No + id: 2835e45d-c778-468c-9ee7-bbd5dc8a7a92 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences (choice order reversed) + reference: '' + ac557360-93dd-4dc1-a393-9c1bf3907af4: !Template + answer_choices: Yes ||| No + id: ac557360-93dd-4dc1-a393-9c1bf3907af4 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' + b8137185-8581-4aaa-a2f4-cc7d48e773ca: !Template answer_choices: null - id: a4b6ff3c-a6f9-49c6-9e5e-dee4d1505d25 + id: b8137185-8581-4aaa-a2f4-cc7d48e773ca jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -90,38 +109,69 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: random_choice_A_B_2 reference: '' - aae78eb4-d350-4feb-9745-84b1b6d06754: !Template - answer_choices: null - id: aae78eb4-d350-4feb-9745-84b1b6d06754 + c2f25416-e628-4123-a6cc-8429345fd6f0: !Template + answer_choices: Yes ||| No + id: c2f25416-e628-4123-a6cc-8429345fd6f0 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences (choice order reversed) + reference: '' + d3e7d4b7-6c35-450a-aafa-f9f60dffaeff: !Template + answer_choices: Yes ||| No + id: d3e7d4b7-6c35-450a-aafa-f9f60dffaeff + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' + e1322160-dcc6-4b8a-975f-86965799505a: !Template + answer_choices: A ||| B + id: e1322160-dcc6-4b8a-975f-86965799505a jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} - {% if shuffled_order == 0 %} - - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -129,39 +179,14 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 - reference: '' - baa9feb8-b4e6-4340-a1ef-3729f86a8739: !Template - answer_choices: Yes ||| No - id: baa9feb8-b4e6-4340-a1ef-3729f86a8739 - jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] - }} - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences - reference: '' - be885170-cf6d-4744-9801-6ed98c7f5c74: !Template - answer_choices: A ||| B - id: be885170-cf6d-4744-9801-6ed98c7f5c74 - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'A/B choice 1 (good-bad, answer: A)' + name: A/B choice randomized reference: '' - d8d53f3e-2c75-4b9f-835b-88a1b4b245f8: !Template + ee63121a-8c10-4a19-a8da-080cc36399d5: !Template answer_choices: null - id: d8d53f3e-2c75-4b9f-835b-88a1b4b245f8 + id: ee63121a-8c10-4a19-a8da-080cc36399d5 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -193,43 +218,82 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: random_choice_1_2 reference: '' - e67ca969-466d-413f-af05-1bb448391da8: !Template - answer_choices: B ||| A - id: e67ca969-466d-413f-af05-1bb448391da8 - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] - }}' + f06e7379-09ca-490e-b60b-54f2bdcf9830: !Template + answer_choices: null + id: f06e7379-09ca-490e-b60b-54f2bdcf9830 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'B/A choice 1 (good-bad, answer: A)' - reference: '' - eca41738-860d-40ad-b0a0-be7e5f6418dc: !Template - answer_choices: Yes ||| No - id: eca41738-860d-40ad-b0a0-be7e5f6418dc - jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] - }} - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences + name: random_choice_A_B reference: '' - f2c5595a-98ad-4d4b-b56a-1003f53865c8: !Template - answer_choices: B ||| A - id: f2c5595a-98ad-4d4b-b56a-1003f53865c8 - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] - }}' + f291441f-914d-406a-adbf-8641415ea6c3: !Template + answer_choices: A ||| B + id: f291441f-914d-406a-adbf-8641415ea6c3 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'B/A choice, 2 (bad-good, answer: B)' + name: A/B choice randomized (choice order reversed) reference: '' diff --git a/promptsource/templates/blimp/causative/templates.yaml b/promptsource/templates/blimp/causative/templates.yaml index 4911a649a..d7d2ec6e7 100644 --- a/promptsource/templates/blimp/causative/templates.yaml +++ b/promptsource/templates/blimp/causative/templates.yaml @@ -1,75 +1,118 @@ dataset: blimp subset: causative templates: - 1052f205-1507-4e92-854c-720eac6eab0e: !Template - answer_choices: B ||| A - id: 1052f205-1507-4e92-854c-720eac6eab0e - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] - }}' + 15b8261d-8fb8-4977-8e02-ebc6c82974f7: !Template + answer_choices: null + id: 15b8261d-8fb8-4977-8e02-ebc6c82974f7 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'B/A choice, 2 (bad-good, answer: B)' - reference: '' - 3fe6c960-93b4-4754-a55c-df63c6ab420b: !Template - answer_choices: Yes ||| No - id: 3fe6c960-93b4-4754-a55c-df63c6ab420b - jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] - }} - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences + name: random_choice_A_B reference: '' - 4f9a24d2-b41e-409d-950b-0c16071b824c: !Template - answer_choices: A ||| B - id: 4f9a24d2-b41e-409d-950b-0c16071b824c - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] - }}' + 5b8633f8-b78f-4798-8a2d-d36d5863cc09: !Template + answer_choices: null + id: 5b8633f8-b78f-4798-8a2d-d36d5863cc09 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'A/B choice 2 (bad-good, answer: B)' + name: random_choice_1_2_2 reference: '' - 68ec16bd-a092-4925-b3df-4911a2f385c1: !Template - answer_choices: A ||| B - id: 68ec16bd-a092-4925-b3df-4911a2f385c1 - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] - }}' + 8293e16e-3edd-4ff3-8e6f-c4fd24ca58a2: !Template + answer_choices: Yes ||| No + id: 8293e16e-3edd-4ff3-8e6f-c4fd24ca58a2 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'A/B choice 1 (good-bad, answer: A)' + original_task: false + name: Yes/No binary judgment for bad sentences (choice order reversed) reference: '' - 85053740-aceb-4df6-b5e5-8ee9d60d7abf: !Template + 97b44339-53be-4e33-a84e-4466eb5688f4: !Template answer_choices: null - id: 85053740-aceb-4df6-b5e5-8ee9d60d7abf + id: 97b44339-53be-4e33-a84e-4466eb5688f4 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -77,11 +120,11 @@ templates: {% if shuffled_order == 0 %} - A + 1 {% else %} - B + 2 {% endif %}' metadata: !TemplateMetadata @@ -89,50 +132,69 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: random_choice_1_2 reference: '' - 85cdc805-415e-462b-b582-847140fae237: !Template + 9bfe4827-d68e-44e1-b465-20240f9c3e1d: !Template answer_choices: Yes ||| No - id: 85cdc805-415e-462b-b582-847140fae237 - jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] - }} + id: 9bfe4827-d68e-44e1-b465-20240f9c3e1d + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata - choices_in_prompt: false + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences (choice order reversed) + reference: '' + 9edceb34-f71e-48e1-9d9e-838c0b4087ec: !Template + answer_choices: Yes ||| No + id: 9edceb34-f71e-48e1-9d9e-838c0b4087ec + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true metrics: - Accuracy original_task: false name: Yes/No binary judgment for good sentences reference: '' - 93dd6231-2d92-43e4-9feb-fc1faec0b594: !Template - answer_choices: null - id: 93dd6231-2d92-43e4-9feb-fc1faec0b594 + a236ff5d-831c-4bab-a709-0a949e3497d7: !Template + answer_choices: A ||| B + id: a236ff5d-831c-4bab-a709-0a949e3497d7 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -140,38 +202,37 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: A/B choice randomized reference: '' - b7fc3d16-37d1-4077-9909-98345754e7ef: !Template - answer_choices: null - id: b7fc3d16-37d1-4077-9909-98345754e7ef + a8472746-14c2-410e-8122-82610156baef: !Template + answer_choices: A ||| B + id: a8472746-14c2-410e-8122-82610156baef jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} - {% if shuffled_order == 0 %} - - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -179,26 +240,26 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: A/B choice randomized (choice order reversed) reference: '' - cb4cede4-02f7-48ed-87d3-88ff805b8448: !Template + aa86f11c-cc83-4748-a160-d5b5e8ac7417: !Template answer_choices: null - id: cb4cede4-02f7-48ed-87d3-88ff805b8448 + id: aa86f11c-cc83-4748-a160-d5b5e8ac7417 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -206,11 +267,11 @@ templates: {% if shuffled_order == 0 %} - 1 + A {% else %} - 2 + B {% endif %}' metadata: !TemplateMetadata @@ -218,18 +279,21 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: random_choice_A_B_2 reference: '' - e880242a-e482-4fd8-8108-a0882763ee99: !Template - answer_choices: B ||| A - id: e880242a-e482-4fd8-8108-a0882763ee99 - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] - }}' + f3fe2424-b32d-461c-9745-9e833fcb695b: !Template + answer_choices: Yes ||| No + id: f3fe2424-b32d-461c-9745-9e833fcb695b + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'B/A choice 1 (good-bad, answer: A)' + original_task: false + name: Yes/No binary judgment for bad sentences reference: '' diff --git a/promptsource/templates/blimp/complex_NP_island/templates.yaml b/promptsource/templates/blimp/complex_NP_island/templates.yaml index 88f3d8e2c..f16e65bd0 100644 --- a/promptsource/templates/blimp/complex_NP_island/templates.yaml +++ b/promptsource/templates/blimp/complex_NP_island/templates.yaml @@ -1,25 +1,82 @@ dataset: blimp subset: complex_NP_island templates: - 2607ed18-bd42-4f19-bbaf-a23b7558e7e4: !Template - answer_choices: B ||| A - id: 2607ed18-bd42-4f19-bbaf-a23b7558e7e4 - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] - }}' + 346563c3-4bd2-4c82-83a9-8e50a158aca4: !Template + answer_choices: A ||| B + id: 346563c3-4bd2-4c82-83a9-8e50a158aca4 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'B/A choice 1 (good-bad, answer: A)' + name: A/B choice randomized (choice order reversed) reference: '' - 2cb9d1cf-0867-4c17-a845-d5737dcf2e94: !Template + 3497056c-4444-44d6-be50-a66231a4f4d3: !Template + answer_choices: Yes ||| No + id: 3497056c-4444-44d6-be50-a66231a4f4d3 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' + 6a8a5b4b-dec7-4f23-a38c-804555c233aa: !Template + answer_choices: Yes ||| No + id: 6a8a5b4b-dec7-4f23-a38c-804555c233aa + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' + ab5d533c-ea31-483f-9c3b-1c60cb8a4d2f: !Template answer_choices: null - id: 2cb9d1cf-0867-4c17-a845-d5737dcf2e94 + id: ab5d533c-ea31-483f-9c3b-1c60cb8a4d2f jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -51,24 +108,27 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: random_choice_A_B_2 reference: '' - 6b647b69-c034-4f5d-aaec-0997f60f456e: !Template - answer_choices: A ||| B - id: 6b647b69-c034-4f5d-aaec-0997f60f456e - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] - }}' + c231307c-5c87-4183-8222-077840a7aa02: !Template + answer_choices: Yes ||| No + id: c231307c-5c87-4183-8222-077840a7aa02 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'A/B choice 2 (bad-good, answer: B)' + original_task: false + name: Yes/No binary judgment for bad sentences (choice order reversed) reference: '' - 6caa4ee8-bb37-46bc-b4e2-f7af9f5f7120: !Template + cdc7aef9-23ac-4993-8f60-d7080c785a04: !Template answer_choices: null - id: 6caa4ee8-bb37-46bc-b4e2-f7af9f5f7120 + id: cdc7aef9-23ac-4993-8f60-d7080c785a04 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -105,49 +165,51 @@ templates: original_task: true name: random_choice_1_2_2 reference: '' - 7a2a5800-3ea2-4171-952d-8351a6cee375: !Template - answer_choices: A ||| B - id: 7a2a5800-3ea2-4171-952d-8351a6cee375 - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] - }}' + d7f278bc-4d8a-4498-a6b5-580f1c84d7bc: !Template + answer_choices: Yes ||| No + id: d7f278bc-4d8a-4498-a6b5-580f1c84d7bc + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'A/B choice 1 (good-bad, answer: A)' + original_task: false + name: Yes/No binary judgment for good sentences (choice order reversed) reference: '' - 88626f98-028c-472c-9f82-00a9b99c36dd: !Template - answer_choices: null - id: 88626f98-028c-472c-9f82-00a9b99c36dd + dc307bdb-7a73-41ca-9b0c-09736dd3d042: !Template + answer_choices: A ||| B + id: dc307bdb-7a73-41ca-9b0c-09736dd3d042 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -155,39 +217,14 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 - reference: '' - 8e6c7628-a338-4b21-ab0e-7d3004596674: !Template - answer_choices: B ||| A - id: 8e6c7628-a338-4b21-ab0e-7d3004596674 - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'B/A choice, 2 (bad-good, answer: B)' - reference: '' - b9767247-12a9-4889-8009-eb87f59b7f77: !Template - answer_choices: Yes ||| No - id: b9767247-12a9-4889-8009-eb87f59b7f77 - jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] - }} - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences + name: A/B choice randomized reference: '' - d52411a5-e1b0-4ef4-ae59-7e470884477d: !Template + e852ac7d-f2ca-456e-a32d-931fcd552081: !Template answer_choices: null - id: d52411a5-e1b0-4ef4-ae59-7e470884477d + id: e852ac7d-f2ca-456e-a32d-931fcd552081 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -219,17 +256,44 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: random_choice_A_B reference: '' - d6d4a7f1-db0e-441b-8f6c-33333926e630: !Template - answer_choices: Yes ||| No - id: d6d4a7f1-db0e-441b-8f6c-33333926e630 - jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] - }} + fe52bb02-c6c7-4687-8d99-ed9d7bdc48d8: !Template + answer_choices: null + id: fe52bb02-c6c7-4687-8d99-ed9d7bdc48d8 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' metadata: !TemplateMetadata - choices_in_prompt: false + choices_in_prompt: true metrics: - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences + original_task: true + name: random_choice_1_2 reference: '' diff --git a/promptsource/templates/blimp/coordinate_structure_constraint_complex_left_branch/templates.yaml b/promptsource/templates/blimp/coordinate_structure_constraint_complex_left_branch/templates.yaml index 886296625..7e21d99c7 100644 --- a/promptsource/templates/blimp/coordinate_structure_constraint_complex_left_branch/templates.yaml +++ b/promptsource/templates/blimp/coordinate_structure_constraint_complex_left_branch/templates.yaml @@ -1,25 +1,12 @@ dataset: blimp subset: coordinate_structure_constraint_complex_left_branch templates: - 129e334d-48b1-4613-8b53-9be2dbe216a4: !Template - answer_choices: B ||| A - id: 129e334d-48b1-4613-8b53-9be2dbe216a4 - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'B/A choice, 2 (bad-good, answer: B)' - reference: '' - 3422f883-6719-40be-aa3d-1ccb186db977: !Template + 0121e488-e96b-4a16-897e-18e36d1ea63a: !Template answer_choices: null - id: 3422f883-6719-40be-aa3d-1ccb186db977 + id: 0121e488-e96b-4a16-897e-18e36d1ea63a jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -51,14 +38,14 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: random_choice_A_B_2 reference: '' - 3d6fb365-6d4e-4b4f-8ec4-389a2c621072: !Template + 36f72863-4705-4fb7-ab70-d6ac1481dde1: !Template answer_choices: null - id: 3d6fb365-6d4e-4b4f-8ec4-389a2c621072 + id: 36f72863-4705-4fb7-ab70-d6ac1481dde1 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -90,14 +77,90 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: random_choice_1_2_2 + reference: '' + 7ac016b3-2d12-463d-9e0f-703b6cbe0a91: !Template + answer_choices: A ||| B + id: 7ac016b3-2d12-463d-9e0f-703b6cbe0a91 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: A/B choice randomized (choice order reversed) + reference: '' + 7f50cf6a-ca05-4815-b1a3-e04f93b00ac7: !Template + answer_choices: A ||| B + id: 7f50cf6a-ca05-4815-b1a3-e04f93b00ac7 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: A/B choice randomized reference: '' - 3dbe2f72-0956-474f-a5c4-35ca59d6eb71: !Template + ade9109e-db39-4328-b259-a6ab965e5cfd: !Template answer_choices: null - id: 3dbe2f72-0956-474f-a5c4-35ca59d6eb71 + id: ade9109e-db39-4328-b259-a6ab965e5cfd jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -129,52 +192,46 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: random_choice_1_2 reference: '' - 440c43d8-bdca-490a-aca8-b10dba9e6ca1: !Template + afb4a824-cc46-45ac-bc3c-3c43d000f947: !Template answer_choices: Yes ||| No - id: 440c43d8-bdca-490a-aca8-b10dba9e6ca1 - jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] - }} + id: afb4a824-cc46-45ac-bc3c-3c43d000f947 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata - choices_in_prompt: false + choices_in_prompt: true metrics: - Accuracy original_task: false name: Yes/No binary judgment for good sentences reference: '' - 8e1b0404-e761-401a-b0b4-204a4c8018b2: !Template - answer_choices: A ||| B - id: 8e1b0404-e761-401a-b0b4-204a4c8018b2 - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'A/B choice 1 (good-bad, answer: A)' - reference: '' - 9f591c11-2ea8-4d4d-95df-f3cbae9f39af: !Template - answer_choices: A ||| B - id: 9f591c11-2ea8-4d4d-95df-f3cbae9f39af - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] - }}' + c0d96488-8976-4594-bc06-0cef9239081d: !Template + answer_choices: Yes ||| No + id: c0d96488-8976-4594-bc06-0cef9239081d + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'A/B choice 2 (bad-good, answer: B)' + original_task: false + name: Yes/No binary judgment for good sentences (choice order reversed) reference: '' - a61e0505-cf82-4239-855a-b79352b2ee94: !Template + cf445379-d6fe-4d46-86a3-8155f2101cae: !Template answer_choices: null - id: a61e0505-cf82-4239-855a-b79352b2ee94 + id: cf445379-d6fe-4d46-86a3-8155f2101cae jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -206,28 +263,35 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: random_choice_A_B reference: '' - b90db41b-150f-422e-8bda-cfdab1443958: !Template - answer_choices: B ||| A - id: b90db41b-150f-422e-8bda-cfdab1443958 - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] - }}' + ea8fd446-0b09-4f76-8b49-06fb6e99e074: !Template + answer_choices: Yes ||| No + id: ea8fd446-0b09-4f76-8b49-06fb6e99e074 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'B/A choice 1 (good-bad, answer: A)' + original_task: false + name: Yes/No binary judgment for bad sentences (choice order reversed) reference: '' - ce927c2d-b220-459e-8382-44740d923591: !Template + f861c143-c477-403a-b3ca-c051a55d374b: !Template answer_choices: Yes ||| No - id: ce927c2d-b220-459e-8382-44740d923591 - jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] - }} + id: f861c143-c477-403a-b3ca-c051a55d374b + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata - choices_in_prompt: false + choices_in_prompt: true metrics: - Accuracy original_task: false diff --git a/promptsource/templates/blimp/coordinate_structure_constraint_object_extraction/templates.yaml b/promptsource/templates/blimp/coordinate_structure_constraint_object_extraction/templates.yaml index 667c9f0a6..6861a17e5 100644 --- a/promptsource/templates/blimp/coordinate_structure_constraint_object_extraction/templates.yaml +++ b/promptsource/templates/blimp/coordinate_structure_constraint_object_extraction/templates.yaml @@ -1,22 +1,9 @@ dataset: blimp subset: coordinate_structure_constraint_object_extraction templates: - 04122a9b-de72-44e0-9d5a-02e4a789c6ad: !Template - answer_choices: B ||| A - id: 04122a9b-de72-44e0-9d5a-02e4a789c6ad - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'B/A choice 1 (good-bad, answer: A)' - reference: '' - 04581e94-4cc7-45ba-a986-01858160756c: !Template + 0f41865d-bfe4-4664-964d-fee06008f421: !Template answer_choices: null - id: 04581e94-4cc7-45ba-a986-01858160756c + id: 0f41865d-bfe4-4664-964d-fee06008f421 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer 1 or 2. @@ -53,59 +40,102 @@ templates: original_task: true name: random_choice_1_2 reference: '' - 15f14103-d891-4f38-812f-ae0804deab11: !Template - answer_choices: Yes ||| No - id: 15f14103-d891-4f38-812f-ae0804deab11 - jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] - }} - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences - reference: '' - 221ccc0d-cfe8-4d0f-be60-42047cc0f71c: !Template + 20db9e37-ca02-4112-abec-3923e84aa0e2: !Template answer_choices: A ||| B - id: 221ccc0d-cfe8-4d0f-be60-42047cc0f71c - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] - }}' + id: 20db9e37-ca02-4112-abec-3923e84aa0e2 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'A/B choice 1 (good-bad, answer: A)' + name: A/B choice randomized reference: '' - 43be39a9-9bc2-4951-a377-d82ffa138780: !Template - answer_choices: B ||| A - id: 43be39a9-9bc2-4951-a377-d82ffa138780 - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] - }}' + 5b38757a-3df9-4ffe-a147-3a3c6c4347ed: !Template + answer_choices: null + id: 5b38757a-3df9-4ffe-a147-3a3c6c4347ed + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'B/A choice, 2 (bad-good, answer: B)' + name: random_choice_A_B_2 reference: '' - 9097e81f-adfa-474d-a35d-e9faaa600559: !Template + 878a9e21-a6bc-4675-bde0-73934614f31d: !Template answer_choices: Yes ||| No - id: 9097e81f-adfa-474d-a35d-e9faaa600559 - jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] - }} + id: 878a9e21-a6bc-4675-bde0-73934614f31d + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata - choices_in_prompt: false + choices_in_prompt: true metrics: - Accuracy original_task: false - name: Yes/No binary judgment for bad sentences + name: Yes/No binary judgment for good sentences reference: '' - 9f1d7348-9bde-4083-812a-52567a95f2ea: !Template + 8b7f1f3d-f2ac-4201-b3b5-201ef9c2bd7b: !Template answer_choices: null - id: 9f1d7348-9bde-4083-812a-52567a95f2ea + id: 8b7f1f3d-f2ac-4201-b3b5-201ef9c2bd7b jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -142,12 +172,60 @@ templates: original_task: true name: random_choice_1_2_2 reference: '' - cc32ed2f-2c2f-476d-9fae-53efac1921f1: !Template + a1c8f9df-d44f-4743-9466-55b49c51382a: !Template + answer_choices: Yes ||| No + id: a1c8f9df-d44f-4743-9466-55b49c51382a + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences (choice order reversed) + reference: '' + a6f76be1-a09b-4dd2-913e-6c1d9e28548c: !Template + answer_choices: Yes ||| No + id: a6f76be1-a09b-4dd2-913e-6c1d9e28548c + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences (choice order reversed) + reference: '' + ad8ff748-0507-4f8f-857e-dfcc530e45d9: !Template + answer_choices: Yes ||| No + id: ad8ff748-0507-4f8f-857e-dfcc530e45d9 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' + aecd346a-4ce9-46dd-b69c-32c02b37e076: !Template answer_choices: null - id: cc32ed2f-2c2f-476d-9fae-53efac1921f1 + id: aecd346a-4ce9-46dd-b69c-32c02b37e076 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -179,51 +257,37 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: random_choice_A_B reference: '' - f54867ae-1231-4c3a-b7e9-f92ad2f4c2fe: !Template + f8780981-6bef-41f4-bbc1-c1ecb0267bf7: !Template answer_choices: A ||| B - id: f54867ae-1231-4c3a-b7e9-f92ad2f4c2fe - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'A/B choice 2 (bad-good, answer: B)' - reference: '' - f5f9f535-3082-4efe-9fa2-8c65ff8977c6: !Template - answer_choices: null - id: f5f9f535-3082-4efe-9fa2-8c65ff8977c6 + id: f8780981-6bef-41f4-bbc1-c1ecb0267bf7 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} - {% if shuffled_order == 0 %} - - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -231,5 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: A/B choice randomized (choice order reversed) reference: '' diff --git a/promptsource/templates/blimp/determiner_noun_agreement_1/templates.yaml b/promptsource/templates/blimp/determiner_noun_agreement_1/templates.yaml index aa0ec68fc..0658610e0 100644 --- a/promptsource/templates/blimp/determiner_noun_agreement_1/templates.yaml +++ b/promptsource/templates/blimp/determiner_noun_agreement_1/templates.yaml @@ -1,12 +1,28 @@ dataset: blimp subset: determiner_noun_agreement_1 templates: - 0fd3119b-070a-4b76-93ed-45d1383dffd4: !Template + 16a971e1-163d-46e5-985f-629751cc71c3: !Template + answer_choices: Yes ||| No + id: 16a971e1-163d-46e5-985f-629751cc71c3 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' + 23c29387-dac6-4799-b045-9984c2c4982d: !Template answer_choices: null - id: 0fd3119b-070a-4b76-93ed-45d1383dffd4 + id: 23c29387-dac6-4799-b045-9984c2c4982d jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -38,14 +54,52 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: random_choice_1_2_2 + reference: '' + 659d61d8-c28e-4e4a-90ab-f6acb5556141: !Template + answer_choices: A ||| B + id: 659d61d8-c28e-4e4a-90ab-f6acb5556141 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: A/B choice randomized (choice order reversed) reference: '' - 194aa45a-aaf0-42be-8858-f2cb83c851ec: !Template + 6a17b03d-92e0-4d4e-9b56-49fc6be6b15c: !Template answer_choices: null - id: 194aa45a-aaf0-42be-8858-f2cb83c851ec + id: 6a17b03d-92e0-4d4e-9b56-49fc6be6b15c jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -77,39 +131,14 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 - reference: '' - 280ae1d0-e23d-4b96-8fe6-cf7cba71f56a: !Template - answer_choices: Yes ||| No - id: 280ae1d0-e23d-4b96-8fe6-cf7cba71f56a - jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] - }} - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences - reference: '' - 44daca2a-6c53-4644-93fa-84b938e7ae58: !Template - answer_choices: A ||| B - id: 44daca2a-6c53-4644-93fa-84b938e7ae58 - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'A/B choice 1 (good-bad, answer: A)' + name: random_choice_1_2 reference: '' - 657044d9-cb52-4c2b-b35c-423b5e3aaefc: !Template + 78169e75-93f7-4163-be6e-1fb4f253a6ba: !Template answer_choices: null - id: 657044d9-cb52-4c2b-b35c-423b5e3aaefc + id: 78169e75-93f7-4163-be6e-1fb4f253a6ba jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -141,51 +170,53 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: random_choice_A_B reference: '' - 665b5c4c-7919-4d2b-9b37-666e0631e04c: !Template - answer_choices: B ||| A - id: 665b5c4c-7919-4d2b-9b37-666e0631e04c - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] - }}' + 8265598d-6ee0-4626-9bea-e09df0bce1f8: !Template + answer_choices: Yes ||| No + id: 8265598d-6ee0-4626-9bea-e09df0bce1f8 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'B/A choice, 2 (bad-good, answer: B)' + original_task: false + name: Yes/No binary judgment for bad sentences (choice order reversed) reference: '' - 9f3e846a-217d-41a4-a6be-1fa0165996f4: !Template - answer_choices: null - id: 9f3e846a-217d-41a4-a6be-1fa0165996f4 + a07d22fb-6092-4448-902a-ea678f72cc32: !Template + answer_choices: A ||| B + id: a07d22fb-6092-4448-902a-ea678f72cc32 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} - {% if shuffled_order == 0 %} - - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -193,41 +224,74 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: A/B choice randomized reference: '' - a3f470e9-2493-4a9c-8a9b-713b002e62ff: !Template - answer_choices: A ||| B - id: a3f470e9-2493-4a9c-8a9b-713b002e62ff - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] - }}' + ad08907e-ae6b-48e1-80d9-efba35f9a0b8: !Template + answer_choices: Yes ||| No + id: ad08907e-ae6b-48e1-80d9-efba35f9a0b8 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'A/B choice 2 (bad-good, answer: B)' + original_task: false + name: Yes/No binary judgment for good sentences (choice order reversed) reference: '' - cbbea246-35e8-4d7b-b640-26b18b9ac42c: !Template - answer_choices: B ||| A - id: cbbea246-35e8-4d7b-b640-26b18b9ac42c - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] - }}' + ea27c898-7d78-4f03-a96f-4f045a9f8a7f: !Template + answer_choices: null + id: ea27c898-7d78-4f03-a96f-4f045a9f8a7f + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'B/A choice 1 (good-bad, answer: A)' + name: random_choice_A_B_2 reference: '' - df12f192-94b8-4d13-a129-da78e725adf5: !Template + f3e5a028-2344-4e21-b200-47da95bf028c: !Template answer_choices: Yes ||| No - id: df12f192-94b8-4d13-a129-da78e725adf5 - jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] - }} + id: f3e5a028-2344-4e21-b200-47da95bf028c + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata - choices_in_prompt: false + choices_in_prompt: true metrics: - Accuracy original_task: false diff --git a/promptsource/templates/blimp/determiner_noun_agreement_2/templates.yaml b/promptsource/templates/blimp/determiner_noun_agreement_2/templates.yaml index 9bc4229cb..bad42ae27 100644 --- a/promptsource/templates/blimp/determiner_noun_agreement_2/templates.yaml +++ b/promptsource/templates/blimp/determiner_noun_agreement_2/templates.yaml @@ -1,24 +1,24 @@ dataset: blimp subset: determiner_noun_agreement_2 templates: - 223e4cb1-3eec-47a0-846f-1b4cd02446b5: !Template + 0339ae7d-0373-4368-b8dd-da73930d6426: !Template answer_choices: null - id: 223e4cb1-3eec-47a0-846f-1b4cd02446b5 + id: 0339ae7d-0373-4368-b8dd-da73930d6426 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -26,11 +26,11 @@ templates: {% if shuffled_order == 0 %} - 1 + A {% else %} - 2 + B {% endif %}' metadata: !TemplateMetadata @@ -38,26 +38,42 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: random_choice_A_B + reference: '' + 12186013-27b2-4e7f-aba7-0c365edcc708: !Template + answer_choices: Yes ||| No + id: 12186013-27b2-4e7f-aba7-0c365edcc708 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences reference: '' - 269cc713-a4bf-4031-bb61-4ba07df76199: !Template + 29ab2bf4-22ba-4bf7-a234-c6d8fc4d6fb2: !Template answer_choices: null - id: 269cc713-a4bf-4031-bb61-4ba07df76199 + id: 29ab2bf4-22ba-4bf7-a234-c6d8fc4d6fb2 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -65,11 +81,11 @@ templates: {% if shuffled_order == 0 %} - A + 1 {% else %} - B + 2 {% endif %}' metadata: !TemplateMetadata @@ -77,77 +93,52 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: random_choice_1_2 reference: '' - 61d53280-467e-4d8a-a79d-ff5c4a5586dc: !Template + 49ed8ab5-946c-47a7-b3fe-d718e07d3cda: !Template answer_choices: A ||| B - id: 61d53280-467e-4d8a-a79d-ff5c4a5586dc - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'A/B choice 2 (bad-good, answer: B)' - reference: '' - 633df725-56db-4a6e-9a2e-e352f08c7b9e: !Template - answer_choices: B ||| A - id: 633df725-56db-4a6e-9a2e-e352f08c7b9e - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'B/A choice 1 (good-bad, answer: A)' - reference: '' - 974b3a6d-283f-4758-81f7-bfa5a75c4006: !Template - answer_choices: Yes ||| No - id: 974b3a6d-283f-4758-81f7-bfa5a75c4006 - jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] - }} - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences - reference: '' - 9be37f75-18d4-4a16-b214-36c7c6813380: !Template - answer_choices: B ||| A - id: 9be37f75-18d4-4a16-b214-36c7c6813380 - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] - }}' + id: 49ed8ab5-946c-47a7-b3fe-d718e07d3cda + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'B/A choice, 2 (bad-good, answer: B)' - reference: '' - 9dc0142e-b289-4fdd-a33e-67fbce65f7ee: !Template - answer_choices: Yes ||| No - id: 9dc0142e-b289-4fdd-a33e-67fbce65f7ee - jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] - }} - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences + name: A/B choice randomized (choice order reversed) reference: '' - cf8187e8-90ed-46fe-ba3e-68ed6f094547: !Template + 86889d8d-79bd-423c-b0d7-746032d59aac: !Template answer_choices: null - id: cf8187e8-90ed-46fe-ba3e-68ed6f094547 + id: 86889d8d-79bd-423c-b0d7-746032d59aac jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -179,27 +170,14 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 - reference: '' - d5e30d96-961c-4936-a5c3-3e802454f4bf: !Template - answer_choices: A ||| B - id: d5e30d96-961c-4936-a5c3-3e802454f4bf - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'A/B choice 1 (good-bad, answer: A)' + name: random_choice_1_2_2 reference: '' - fb6a991a-cce6-4ed3-a067-06ff84488bbe: !Template + 8e348173-4835-4a2b-b0ba-cdb0711cf101: !Template answer_choices: null - id: fb6a991a-cce6-4ed3-a067-06ff84488bbe + id: 8e348173-4835-4a2b-b0ba-cdb0711cf101 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -231,5 +209,91 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: random_choice_A_B_2 + reference: '' + 9208808a-ce6a-4777-aae8-974c361b34c3: !Template + answer_choices: Yes ||| No + id: 9208808a-ce6a-4777-aae8-974c361b34c3 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences (choice order reversed) + reference: '' + 9acaf85d-e6e8-4d8c-82db-340b70fc8eee: !Template + answer_choices: Yes ||| No + id: 9acaf85d-e6e8-4d8c-82db-340b70fc8eee + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences (choice order reversed) + reference: '' + d1c90a72-23d2-4359-b5de-7328a799d217: !Template + answer_choices: A ||| B + id: d1c90a72-23d2-4359-b5de-7328a799d217 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: A/B choice randomized + reference: '' + d53914cb-8544-48b7-98f2-b89e0f17003d: !Template + answer_choices: Yes ||| No + id: d53914cb-8544-48b7-98f2-b89e0f17003d + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences reference: '' diff --git a/promptsource/templates/blimp/determiner_noun_agreement_irregular_1/templates.yaml b/promptsource/templates/blimp/determiner_noun_agreement_irregular_1/templates.yaml index eadfef437..900a518f1 100644 --- a/promptsource/templates/blimp/determiner_noun_agreement_irregular_1/templates.yaml +++ b/promptsource/templates/blimp/determiner_noun_agreement_irregular_1/templates.yaml @@ -1,37 +1,50 @@ dataset: blimp subset: determiner_noun_agreement_irregular_1 templates: - 0f092653-5b45-4dc2-9e5f-7406906cb12d: !Template + 0ebef220-8636-42ab-9cf4-b70067fde695: !Template answer_choices: A ||| B - id: 0f092653-5b45-4dc2-9e5f-7406906cb12d - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] - }}' + id: 0ebef220-8636-42ab-9cf4-b70067fde695 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'A/B choice 2 (bad-good, answer: B)' - reference: '' - 2b12b43a-b56e-40bc-99b0-ea0c61197caa: !Template - answer_choices: Yes ||| No - id: 2b12b43a-b56e-40bc-99b0-ea0c61197caa - jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] - }} - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences + name: A/B choice randomized reference: '' - 64fad99b-1f26-47dc-a376-be3f54f24517: !Template + 15318c27-bda3-4d65-bdd3-38b54b303edb: !Template answer_choices: null - id: 64fad99b-1f26-47dc-a376-be3f54f24517 + id: 15318c27-bda3-4d65-bdd3-38b54b303edb jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -63,27 +76,52 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: random_choice_1_2_2 reference: '' - 780b7420-7712-4a28-a78e-edfcc051ebaf: !Template - answer_choices: B ||| A - id: 780b7420-7712-4a28-a78e-edfcc051ebaf - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] - }}' + 19e42a55-4b59-4f78-a989-2ed1b4b0e35a: !Template + answer_choices: A ||| B + id: 19e42a55-4b59-4f78-a989-2ed1b4b0e35a + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'B/A choice, 2 (bad-good, answer: B)' + name: A/B choice randomized (choice order reversed) reference: '' - 7bebfb37-eb3f-4e16-821d-8ab8e3e97a20: !Template + 3385fbbc-d6db-4859-aea9-c62d80665ec9: !Template answer_choices: null - id: 7bebfb37-eb3f-4e16-821d-8ab8e3e97a20 + id: 3385fbbc-d6db-4859-aea9-c62d80665ec9 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -115,14 +153,62 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: random_choice_A_B_2 + reference: '' + 950220e7-e981-4d1d-b2d0-9e4d3b91ac3b: !Template + answer_choices: Yes ||| No + id: 950220e7-e981-4d1d-b2d0-9e4d3b91ac3b + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences reference: '' - ca54c469-745a-4918-987f-84fd2bf2e1dd: !Template + b123bfc6-a416-49a2-8f08-b01c5601d07d: !Template + answer_choices: Yes ||| No + id: b123bfc6-a416-49a2-8f08-b01c5601d07d + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' + b5bac42b-4947-4759-b717-3c364023ac30: !Template + answer_choices: Yes ||| No + id: b5bac42b-4947-4759-b717-3c364023ac30 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences (choice order reversed) + reference: '' + d6eb4a2f-b5c5-49b1-a6fd-a6538b7c5369: !Template answer_choices: null - id: ca54c469-745a-4918-987f-84fd2bf2e1dd + id: d6eb4a2f-b5c5-49b1-a6fd-a6538b7c5369 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -154,14 +240,30 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: random_choice_1_2 + reference: '' + f484a660-c6e4-4238-92bf-13531f1e9453: !Template + answer_choices: Yes ||| No + id: f484a660-c6e4-4238-92bf-13531f1e9453 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences (choice order reversed) reference: '' - cb6a79a1-8f0b-4762-b8a7-219352fc85f3: !Template + fc625d82-45a2-4d54-a384-72797a29447b: !Template answer_choices: null - id: cb6a79a1-8f0b-4762-b8a7-219352fc85f3 + id: fc625d82-45a2-4d54-a384-72797a29447b jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -193,43 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 - reference: '' - e1066a6d-41ef-4693-8ce9-6e5abe3ad19d: !Template - answer_choices: Yes ||| No - id: e1066a6d-41ef-4693-8ce9-6e5abe3ad19d - jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] - }} - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences - reference: '' - ed1ba6fd-9a83-45a6-9e5c-acbcc42c8543: !Template - answer_choices: B ||| A - id: ed1ba6fd-9a83-45a6-9e5c-acbcc42c8543 - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'B/A choice 1 (good-bad, answer: A)' - reference: '' - fbd32631-90e3-418a-8a8c-430bae7cb8c3: !Template - answer_choices: A ||| B - id: fbd32631-90e3-418a-8a8c-430bae7cb8c3 - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'A/B choice 1 (good-bad, answer: A)' + name: random_choice_A_B reference: '' diff --git a/promptsource/templates/blimp/determiner_noun_agreement_irregular_2/templates.yaml b/promptsource/templates/blimp/determiner_noun_agreement_irregular_2/templates.yaml index 5db2b87a7..d9bf71b9b 100644 --- a/promptsource/templates/blimp/determiner_noun_agreement_irregular_2/templates.yaml +++ b/promptsource/templates/blimp/determiner_noun_agreement_irregular_2/templates.yaml @@ -1,74 +1,41 @@ dataset: blimp subset: determiner_noun_agreement_irregular_2 templates: - 097e2730-3125-4fb0-965d-fefc2f12fc85: !Template - answer_choices: null - id: 097e2730-3125-4fb0-965d-fefc2f12fc85 - jinja: '{% set shuffled_order = [0, 1] | random %} - - Between the two sentences, which one is grammatical? Please answer A or B. - - {% if shuffled_order == 0 %} - - A: {{ sentence_good }} - - B: {{ sentence_bad }} - - {% else %} - - A: {{ sentence_bad }} - - B: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - A - - {% else %} - - B + 0228ca33-f383-47ca-a5bb-f0714bfdc73b: !Template + answer_choices: Yes ||| No + id: 0228ca33-f383-47ca-a5bb-f0714bfdc73b + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - {% endif %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: random_choice_A_B_2 - reference: '' - 19865635-e759-4a45-ba10-ba837c168d1f: !Template - answer_choices: B ||| A - id: 19865635-e759-4a45-ba10-ba837c168d1f - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] - }}' + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'B/A choice, 2 (bad-good, answer: B)' + original_task: false + name: Yes/No binary judgment for bad sentences (choice order reversed) reference: '' - 3a1ce220-3539-41e3-8288-44dad0ac6322: !Template - answer_choices: B ||| A - id: 3a1ce220-3539-41e3-8288-44dad0ac6322 - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] - }}' + 037afad0-dbc5-4cd0-b53e-143ced5ff90e: !Template + answer_choices: Yes ||| No + id: 037afad0-dbc5-4cd0-b53e-143ced5ff90e + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'B/A choice 1 (good-bad, answer: A)' + original_task: false + name: Yes/No binary judgment for good sentences reference: '' - 3f5c5974-3f16-4a1c-993a-970d6a39b5cf: !Template + 3ad8e304-5acc-4f63-b79c-414467d74339: !Template answer_choices: null - id: 3f5c5974-3f16-4a1c-993a-970d6a39b5cf + id: 3ad8e304-5acc-4f63-b79c-414467d74339 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer 1 or 2. @@ -105,9 +72,9 @@ templates: original_task: true name: random_choice_1_2 reference: '' - 4beb8a18-9833-40a7-91a2-79b8c88afa20: !Template + 69a4e6fb-6d82-4daa-9681-9c3aa322afdc: !Template answer_choices: null - id: 4beb8a18-9833-40a7-91a2-79b8c88afa20 + id: 69a4e6fb-6d82-4daa-9681-9c3aa322afdc jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer A or B. @@ -144,9 +111,9 @@ templates: original_task: true name: random_choice_A_B reference: '' - 59c0367d-ac9e-41e7-8a12-919de9aab7fc: !Template + 8a7e61e5-89b2-46bb-9652-04d362b32964: !Template answer_choices: null - id: 59c0367d-ac9e-41e7-8a12-919de9aab7fc + id: 8a7e61e5-89b2-46bb-9652-04d362b32964 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -183,53 +150,150 @@ templates: original_task: true name: random_choice_1_2_2 reference: '' - 7818b45f-0003-49c6-aa61-c50d1b6ae640: !Template + ac822a81-03bf-4327-a1d5-3522fa8ca563: !Template answer_choices: A ||| B - id: 7818b45f-0003-49c6-aa61-c50d1b6ae640 - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] - }}' + id: ac822a81-03bf-4327-a1d5-3522fa8ca563 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'A/B choice 1 (good-bad, answer: A)' + name: A/B choice randomized reference: '' - bc667eda-3467-45f5-baac-f2dc79e33834: !Template + cd4334e0-2f75-4bf7-bed1-e68b462efd86: !Template answer_choices: Yes ||| No - id: bc667eda-3467-45f5-baac-f2dc79e33834 - jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] - }} + id: cd4334e0-2f75-4bf7-bed1-e68b462efd86 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata - choices_in_prompt: false + choices_in_prompt: true metrics: - Accuracy original_task: false - name: Yes/No binary judgment for good sentences + name: Yes/No binary judgment for good sentences (choice order reversed) reference: '' - d88ba508-a5b2-4ffd-a65b-ef7cf57b3066: !Template + d5a5f664-ea12-4b21-bd1b-05990f8f8578: !Template answer_choices: Yes ||| No - id: d88ba508-a5b2-4ffd-a65b-ef7cf57b3066 - jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] - }} + id: d5a5f664-ea12-4b21-bd1b-05990f8f8578 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata - choices_in_prompt: false + choices_in_prompt: true metrics: - Accuracy original_task: false name: Yes/No binary judgment for bad sentences reference: '' - eac5b2f5-db99-48a4-9915-64342d4d0f74: !Template + f5d9ced3-ef82-4f19-a700-42f5c5920ce3: !Template + answer_choices: null + id: f5d9ced3-ef82-4f19-a700-42f5c5920ce3 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B_2 + reference: '' + ff05613a-97f1-4c60-b9e7-8f7e181d46a9: !Template answer_choices: A ||| B - id: eac5b2f5-db99-48a4-9915-64342d4d0f74 - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] - }}' + id: ff05613a-97f1-4c60-b9e7-8f7e181d46a9 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'A/B choice 2 (bad-good, answer: B)' + name: A/B choice randomized (choice order reversed) reference: '' diff --git a/promptsource/templates/blimp/determiner_noun_agreement_with_adj_2/templates.yaml b/promptsource/templates/blimp/determiner_noun_agreement_with_adj_2/templates.yaml index fe68b7541..10776bf84 100644 --- a/promptsource/templates/blimp/determiner_noun_agreement_with_adj_2/templates.yaml +++ b/promptsource/templates/blimp/determiner_noun_agreement_with_adj_2/templates.yaml @@ -1,24 +1,12 @@ dataset: blimp subset: determiner_noun_agreement_with_adj_2 templates: - 13ca673f-eb0e-4ae9-be69-4e41a991ff33: !Template - answer_choices: Yes ||| No - id: 13ca673f-eb0e-4ae9-be69-4e41a991ff33 - jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] - }} - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences - reference: '' - 4914bacd-a781-4b71-b73f-0db38535629b: !Template + 3886eb20-4f0a-45ca-b450-010032068940: !Template answer_choices: null - id: 4914bacd-a781-4b71-b73f-0db38535629b + id: 3886eb20-4f0a-45ca-b450-010032068940 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -50,38 +38,53 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: random_choice_A_B_2 reference: '' - 598ea465-01d3-48ce-9608-18f4829129ca: !Template - answer_choices: null - id: 598ea465-01d3-48ce-9608-18f4829129ca + 478ab9b8-65a0-4fff-9246-56d82dc91d51: !Template + answer_choices: Yes ||| No + id: 478ab9b8-65a0-4fff-9246-56d82dc91d51 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' + 4fc5e43b-0f5b-470f-8cfc-a21ae4aa6d09: !Template + answer_choices: A ||| B + id: 4fc5e43b-0f5b-470f-8cfc-a21ae4aa6d09 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} - {% if shuffled_order == 0 %} - - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -89,14 +92,30 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: A/B choice randomized (choice order reversed) + reference: '' + 5d301e88-44c1-4604-800d-96e5824d6e7e: !Template + answer_choices: Yes ||| No + id: 5d301e88-44c1-4604-800d-96e5824d6e7e + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences (choice order reversed) reference: '' - 5b79d519-9e1d-4235-9a55-9dc67928bafb: !Template + 603299fe-ff70-4158-b31f-6e03828e9de4: !Template answer_choices: null - id: 5b79d519-9e1d-4235-9a55-9dc67928bafb + id: 603299fe-ff70-4158-b31f-6e03828e9de4 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -128,14 +147,14 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: random_choice_A_B reference: '' - 766aa0e2-f2e9-431b-9ff0-54592092696f: !Template + a1e1c3a1-495d-40c5-b051-29d0eef99c39: !Template answer_choices: null - id: 766aa0e2-f2e9-431b-9ff0-54592092696f + id: a1e1c3a1-495d-40c5-b051-29d0eef99c39 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -167,69 +186,114 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 - reference: '' - bc33bb59-e1ef-4fc9-85b5-e47a2b1c62b8: !Template - answer_choices: B ||| A - id: bc33bb59-e1ef-4fc9-85b5-e47a2b1c62b8 - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'B/A choice 1 (good-bad, answer: A)' + name: random_choice_1_2 reference: '' - cfb5185c-6af7-4595-9edb-cee461744fd9: !Template + a8055f9b-20b2-4497-8d0a-9321036d251b: !Template answer_choices: A ||| B - id: cfb5185c-6af7-4595-9edb-cee461744fd9 - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] - }}' + id: a8055f9b-20b2-4497-8d0a-9321036d251b + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'A/B choice 1 (good-bad, answer: A)' + name: A/B choice randomized reference: '' - e1c6e2f4-d204-4c66-899f-a1859c8bc95e: !Template + cbd66cab-5519-4b1a-82f0-472145673157: !Template answer_choices: Yes ||| No - id: e1c6e2f4-d204-4c66-899f-a1859c8bc95e - jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] - }} + id: cbd66cab-5519-4b1a-82f0-472145673157 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata - choices_in_prompt: false + choices_in_prompt: true metrics: - Accuracy original_task: false - name: Yes/No binary judgment for good sentences + name: Yes/No binary judgment for bad sentences reference: '' - f9af577a-73f4-4f7e-94b6-33821626c423: !Template - answer_choices: B ||| A - id: f9af577a-73f4-4f7e-94b6-33821626c423 - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] - }}' + d384d2a2-5f92-4e8a-8e87-a4246bd27621: !Template + answer_choices: Yes ||| No + id: d384d2a2-5f92-4e8a-8e87-a4246bd27621 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'B/A choice, 2 (bad-good, answer: B)' + original_task: false + name: Yes/No binary judgment for good sentences (choice order reversed) reference: '' - fc68020f-df74-4afd-87e4-b7a92a79b87b: !Template - answer_choices: A ||| B - id: fc68020f-df74-4afd-87e4-b7a92a79b87b - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] - }}' + fc3a67ec-f95d-4ee7-9ad8-9749b17c2238: !Template + answer_choices: null + id: fc3a67ec-f95d-4ee7-9ad8-9749b17c2238 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'A/B choice 2 (bad-good, answer: B)' + name: random_choice_1_2_2 reference: '' diff --git a/promptsource/templates/blimp/determiner_noun_agreement_with_adj_irregular_1/templates.yaml b/promptsource/templates/blimp/determiner_noun_agreement_with_adj_irregular_1/templates.yaml index c78b330e4..b3e61f867 100644 --- a/promptsource/templates/blimp/determiner_noun_agreement_with_adj_irregular_1/templates.yaml +++ b/promptsource/templates/blimp/determiner_noun_agreement_with_adj_irregular_1/templates.yaml @@ -1,47 +1,118 @@ dataset: blimp subset: determiner_noun_agreement_with_adj_irregular_1 templates: - 150bd9cf-b9c2-4417-aace-90be474c5ff3: !Template - answer_choices: B ||| A - id: 150bd9cf-b9c2-4417-aace-90be474c5ff3 - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] - }}' + 148ed02a-e0fa-4a39-a0d4-c2cd2859e711: !Template + answer_choices: Yes ||| No + id: 148ed02a-e0fa-4a39-a0d4-c2cd2859e711 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' + 295d19cb-8ee3-4588-b074-b2b97c4e73ff: !Template + answer_choices: null + id: 295d19cb-8ee3-4588-b074-b2b97c4e73ff + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'B/A choice, 2 (bad-good, answer: B)' + name: random_choice_1_2 reference: '' - 18403cac-8fb3-4c8c-b455-e0f79f0f5e6b: !Template + 36588004-dc66-4c53-a053-edd3d51130ab: !Template answer_choices: A ||| B - id: 18403cac-8fb3-4c8c-b455-e0f79f0f5e6b - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] - }}' + id: 36588004-dc66-4c53-a053-edd3d51130ab + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'A/B choice 1 (good-bad, answer: A)' + name: A/B choice randomized reference: '' - 286df295-a069-4d1e-b29d-417b9d39b044: !Template + 62aac670-4c21-4c2d-9855-aada50e253dd: !Template answer_choices: Yes ||| No - id: 286df295-a069-4d1e-b29d-417b9d39b044 - jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] - }} + id: 62aac670-4c21-4c2d-9855-aada50e253dd + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata - choices_in_prompt: false + choices_in_prompt: true metrics: - Accuracy original_task: false - name: Yes/No binary judgment for good sentences + name: Yes/No binary judgment for bad sentences reference: '' - 51581cf6-6f89-4f61-aaed-7886fc2f127c: !Template + 943d13a8-2813-4d7e-8ad8-dab9e6acea25: !Template answer_choices: null - id: 51581cf6-6f89-4f61-aaed-7886fc2f127c + id: 943d13a8-2813-4d7e-8ad8-dab9e6acea25 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer A or B. @@ -78,9 +149,9 @@ templates: original_task: true name: random_choice_A_B_2 reference: '' - 597ec04b-0f83-4401-81e4-00eb67c5f876: !Template + a3e6cb80-06d4-4e22-bd58-249436d13d11: !Template answer_choices: null - id: 597ec04b-0f83-4401-81e4-00eb67c5f876 + id: a3e6cb80-06d4-4e22-bd58-249436d13d11 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer A or B. @@ -117,9 +188,9 @@ templates: original_task: true name: random_choice_A_B reference: '' - 62c66714-fd11-4070-868e-56c5c5089d62: !Template + b4dc51e3-d5db-4791-a5f3-ff82a08b8b16: !Template answer_choices: null - id: 62c66714-fd11-4070-868e-56c5c5089d62 + id: b4dc51e3-d5db-4791-a5f3-ff82a08b8b16 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -156,48 +227,67 @@ templates: original_task: true name: random_choice_1_2_2 reference: '' - 913ace8e-782a-43a9-b9b3-b5117170d119: !Template + b8ac6c90-e8b3-41fe-b2eb-fd01b22c069f: !Template answer_choices: Yes ||| No - id: 913ace8e-782a-43a9-b9b3-b5117170d119 - jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] - }} + id: b8ac6c90-e8b3-41fe-b2eb-fd01b22c069f + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata - choices_in_prompt: false + choices_in_prompt: true metrics: - Accuracy original_task: false - name: Yes/No binary judgment for bad sentences + name: Yes/No binary judgment for bad sentences (choice order reversed) reference: '' - 9f0628cc-504c-4c70-9540-0713774e3d3d: !Template - answer_choices: null - id: 9f0628cc-504c-4c70-9540-0713774e3d3d + f192ada3-cc6e-4249-89fb-e74e2c6c42f1: !Template + answer_choices: Yes ||| No + id: f192ada3-cc6e-4249-89fb-e74e2c6c42f1 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences (choice order reversed) + reference: '' + f8061c99-4d02-4d29-a73b-1bdf50411219: !Template + answer_choices: A ||| B + id: f8061c99-4d02-4d29-a73b-1bdf50411219 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} - {% if shuffled_order == 0 %} - - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -205,31 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 - reference: '' - a0890919-8ee2-4d28-896f-43abb4f0adc8: !Template - answer_choices: A ||| B - id: a0890919-8ee2-4d28-896f-43abb4f0adc8 - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'A/B choice 2 (bad-good, answer: B)' - reference: '' - a676df72-b67a-4c3f-92a3-91a7d77da87c: !Template - answer_choices: B ||| A - id: a676df72-b67a-4c3f-92a3-91a7d77da87c - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'B/A choice 1 (good-bad, answer: A)' + name: A/B choice randomized (choice order reversed) reference: '' diff --git a/promptsource/templates/blimp/determiner_noun_agreement_with_adj_irregular_2/templates.yaml b/promptsource/templates/blimp/determiner_noun_agreement_with_adj_irregular_2/templates.yaml index c7138ee3e..8e686fab5 100644 --- a/promptsource/templates/blimp/determiner_noun_agreement_with_adj_irregular_2/templates.yaml +++ b/promptsource/templates/blimp/determiner_noun_agreement_with_adj_irregular_2/templates.yaml @@ -1,37 +1,94 @@ dataset: blimp subset: determiner_noun_agreement_with_adj_irregular_2 templates: - 232ce8bd-94ad-498d-97dd-262b3ec93353: !Template + 1a736052-ab64-4cfa-b233-d5b84cb37076: !Template answer_choices: A ||| B - id: 232ce8bd-94ad-498d-97dd-262b3ec93353 - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] - }}' + id: 1a736052-ab64-4cfa-b233-d5b84cb37076 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'A/B choice 2 (bad-good, answer: B)' + name: A/B choice randomized + reference: '' + 38508ba7-f900-4193-90c7-dbfaec2cd9c6: !Template + answer_choices: Yes ||| No + id: 38508ba7-f900-4193-90c7-dbfaec2cd9c6 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences (choice order reversed) reference: '' - 32c7010c-bb30-4279-9b29-4badb633afce: !Template + 3e03d8c7-ca52-4644-bb26-6b9b3965582a: !Template + answer_choices: Yes ||| No + id: 3e03d8c7-ca52-4644-bb26-6b9b3965582a + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' + 432cf8db-1236-44db-9d81-d449e7cf8807: !Template answer_choices: null - id: 32c7010c-bb30-4279-9b29-4badb633afce + id: 432cf8db-1236-44db-9d81-d449e7cf8807 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -39,11 +96,11 @@ templates: {% if shuffled_order == 0 %} - 1 + A {% else %} - 2 + B {% endif %}' metadata: !TemplateMetadata @@ -51,11 +108,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: random_choice_A_B reference: '' - 3433b964-9c02-476d-97ca-90398e49172e: !Template + 443abdcb-d535-4373-8250-377f9041ff74: !Template answer_choices: null - id: 3433b964-9c02-476d-97ca-90398e49172e + id: 443abdcb-d535-4373-8250-377f9041ff74 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -92,21 +149,9 @@ templates: original_task: true name: random_choice_1_2_2 reference: '' - 54dd2295-1f1b-4bb0-90bc-e5b14478267f: !Template - answer_choices: Yes ||| No - id: 54dd2295-1f1b-4bb0-90bc-e5b14478267f - jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] - }} - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences - reference: '' - 9eeb1515-b05e-41ba-baf6-b861f1c0adcc: !Template + bb790191-ae53-4a76-a9b2-3f5e6592aade: !Template answer_choices: null - id: 9eeb1515-b05e-41ba-baf6-b861f1c0adcc + id: bb790191-ae53-4a76-a9b2-3f5e6592aade jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer A or B. @@ -143,75 +188,94 @@ templates: original_task: true name: random_choice_A_B_2 reference: '' - ac89930d-5eaa-497d-bc4f-1c2860c4e89e: !Template - answer_choices: B ||| A - id: ac89930d-5eaa-497d-bc4f-1c2860c4e89e - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] - }}' + bd329ec8-ed8c-4a6d-844e-d185bf9c63e6: !Template + answer_choices: Yes ||| No + id: bd329ec8-ed8c-4a6d-844e-d185bf9c63e6 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'B/A choice 1 (good-bad, answer: A)' + original_task: false + name: Yes/No binary judgment for good sentences reference: '' - acda4d45-d598-467f-ae6f-b88a720e80a8: !Template + be89a4c0-3de5-4066-b1a6-c70b809c1b84: !Template answer_choices: A ||| B - id: acda4d45-d598-467f-ae6f-b88a720e80a8 - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'A/B choice 1 (good-bad, answer: A)' - reference: '' - c8bfd8f3-f57c-406d-a60d-d2a7aaa8597e: !Template - answer_choices: B ||| A - id: c8bfd8f3-f57c-406d-a60d-d2a7aaa8597e - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] - }}' + id: be89a4c0-3de5-4066-b1a6-c70b809c1b84 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'B/A choice, 2 (bad-good, answer: B)' + name: A/B choice randomized (choice order reversed) reference: '' - f5e48fb8-5441-48c5-814f-5622bac948f8: !Template + ca9cf663-9ced-44d2-8a09-f4bdc3c34f60: !Template answer_choices: Yes ||| No - id: f5e48fb8-5441-48c5-814f-5622bac948f8 - jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] - }} + id: ca9cf663-9ced-44d2-8a09-f4bdc3c34f60 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata - choices_in_prompt: false + choices_in_prompt: true metrics: - Accuracy original_task: false - name: Yes/No binary judgment for bad sentences + name: Yes/No binary judgment for good sentences (choice order reversed) reference: '' - f7adc7ba-afcc-48ab-9d4f-d5bb93225515: !Template + dc194ba6-f216-4bb9-948d-38d52f1d00d4: !Template answer_choices: null - id: f7adc7ba-afcc-48ab-9d4f-d5bb93225515 + id: dc194ba6-f216-4bb9-948d-38d52f1d00d4 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -219,11 +283,11 @@ templates: {% if shuffled_order == 0 %} - A + 1 {% else %} - B + 2 {% endif %}' metadata: !TemplateMetadata @@ -231,5 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: random_choice_1_2 reference: '' diff --git a/promptsource/templates/blimp/determiner_noun_agreement_with_adjective_1/templates.yaml b/promptsource/templates/blimp/determiner_noun_agreement_with_adjective_1/templates.yaml index 06beb3a74..cb11c4a08 100644 --- a/promptsource/templates/blimp/determiner_noun_agreement_with_adjective_1/templates.yaml +++ b/promptsource/templates/blimp/determiner_noun_agreement_with_adjective_1/templates.yaml @@ -1,12 +1,12 @@ dataset: blimp subset: determiner_noun_agreement_with_adjective_1 templates: - 1d4cb84a-790a-43f1-b145-bafe19d90677: !Template + 14dc5768-494f-4c60-ab44-136f5d2e4b18: !Template answer_choices: null - id: 1d4cb84a-790a-43f1-b145-bafe19d90677 + id: 14dc5768-494f-4c60-ab44-136f5d2e4b18 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -38,26 +38,26 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: random_choice_1_2 reference: '' - 1f60c65a-fb5e-488b-ac3b-9126138bc49b: !Template + 2288835c-890b-497c-98e0-0a6e441e487a: !Template answer_choices: null - id: 1f60c65a-fb5e-488b-ac3b-9126138bc49b + id: 2288835c-890b-497c-98e0-0a6e441e487a jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -65,11 +65,11 @@ templates: {% if shuffled_order == 0 %} - 1 + A {% else %} - 2 + B {% endif %}' metadata: !TemplateMetadata @@ -77,38 +77,85 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: random_choice_A_B_2 reference: '' - 51ba2249-12a3-40b0-a162-bbc6642c61fd: !Template - answer_choices: null - id: 51ba2249-12a3-40b0-a162-bbc6642c61fd + 8ca00c13-f9e1-4f21-a87f-0d901596b15b: !Template + answer_choices: Yes ||| No + id: 8ca00c13-f9e1-4f21-a87f-0d901596b15b + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' + 9e312167-48ae-48ea-9e35-91dd20cc1b99: !Template + answer_choices: Yes ||| No + id: 9e312167-48ae-48ea-9e35-91dd20cc1b99 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences (choice order reversed) + reference: '' + 9eb5c3b0-82de-46c9-a44d-1c62a045876b: !Template + answer_choices: Yes ||| No + id: 9eb5c3b0-82de-46c9-a44d-1c62a045876b + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences (choice order reversed) + reference: '' + a0b1f044-dc67-4f43-9d55-1f738ce93ce5: !Template + answer_choices: A ||| B + id: a0b1f044-dc67-4f43-9d55-1f738ce93ce5 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -116,24 +163,49 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: A/B choice randomized reference: '' - 6a190fe0-e353-43ac-922e-02b95cc8e8c3: !Template + b305581a-7188-4073-90b8-2f11d9ede93a: !Template answer_choices: A ||| B - id: 6a190fe0-e353-43ac-922e-02b95cc8e8c3 - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] - }}' + id: b305581a-7188-4073-90b8-2f11d9ede93a + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'A/B choice 2 (bad-good, answer: B)' + name: A/B choice randomized (choice order reversed) reference: '' - 7a11abee-2d43-40ba-9d77-0714ddea41e3: !Template + cc1dbecb-6c8c-43a3-9508-53b8e37a3d6c: !Template answer_choices: null - id: 7a11abee-2d43-40ba-9d77-0714ddea41e3 + id: cc1dbecb-6c8c-43a3-9508-53b8e37a3d6c jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer A or B. @@ -170,66 +242,58 @@ templates: original_task: true name: random_choice_A_B reference: '' - 7c92f1b1-979e-41c5-918a-c24a0427c5a6: !Template - answer_choices: A ||| B - id: 7c92f1b1-979e-41c5-918a-c24a0427c5a6 - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] - }}' + d441f455-c16a-4daa-98c5-a831bdda0d76: !Template + answer_choices: null + id: d441f455-c16a-4daa-98c5-a831bdda0d76 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'A/B choice 1 (good-bad, answer: A)' + name: random_choice_1_2_2 reference: '' - 8f8db402-b5e7-44d3-942a-7c6f260d0356: !Template + e9831865-c53f-4784-84f1-bd92e5920d27: !Template answer_choices: Yes ||| No - id: 8f8db402-b5e7-44d3-942a-7c6f260d0356 - jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] - }} + id: e9831865-c53f-4784-84f1-bd92e5920d27 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata - choices_in_prompt: false + choices_in_prompt: true metrics: - Accuracy original_task: false name: Yes/No binary judgment for good sentences reference: '' - cd6f0265-e95b-4aa3-9596-fb17f8a5205e: !Template - answer_choices: Yes ||| No - id: cd6f0265-e95b-4aa3-9596-fb17f8a5205e - jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] - }} - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences - reference: '' - f95ccc81-68ee-45de-9f20-30b1a8edfe02: !Template - answer_choices: B ||| A - id: f95ccc81-68ee-45de-9f20-30b1a8edfe02 - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'B/A choice 1 (good-bad, answer: A)' - reference: '' - fe6e92d5-7f2d-4155-a90f-9800adb3deb9: !Template - answer_choices: B ||| A - id: fe6e92d5-7f2d-4155-a90f-9800adb3deb9 - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'B/A choice, 2 (bad-good, answer: B)' - reference: '' diff --git a/promptsource/templates/blimp/distractor_agreement_relational_noun/templates.yaml b/promptsource/templates/blimp/distractor_agreement_relational_noun/templates.yaml index 5ca416469..90a06ebc2 100644 --- a/promptsource/templates/blimp/distractor_agreement_relational_noun/templates.yaml +++ b/promptsource/templates/blimp/distractor_agreement_relational_noun/templates.yaml @@ -1,22 +1,63 @@ dataset: blimp subset: distractor_agreement_relational_noun templates: - 07170e39-a4cd-4fec-95a0-29c37e8d6504: !Template - answer_choices: B ||| A - id: 07170e39-a4cd-4fec-95a0-29c37e8d6504 - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] - }}' + 4c188be7-ba6a-4555-8b3b-329a9586aa58: !Template + answer_choices: Yes ||| No + id: 4c188be7-ba6a-4555-8b3b-329a9586aa58 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' + 52f83bdc-0af7-407f-8eb2-30f0b82b0861: !Template + answer_choices: A ||| B + id: 52f83bdc-0af7-407f-8eb2-30f0b82b0861 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'B/A choice 1 (good-bad, answer: A)' + name: A/B choice randomized (choice order reversed) reference: '' - 33f9426d-25f5-420f-ba89-8b420020861d: !Template + 79712317-8ceb-4dda-98b5-2ede1f561f1d: !Template answer_choices: null - id: 33f9426d-25f5-420f-ba89-8b420020861d + id: 79712317-8ceb-4dda-98b5-2ede1f561f1d jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer A or B. @@ -53,37 +94,72 @@ templates: original_task: true name: random_choice_A_B reference: '' - 39b4ee35-30db-4618-bbc8-0327a12044e4: !Template - answer_choices: A ||| B - id: 39b4ee35-30db-4618-bbc8-0327a12044e4 - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] - }}' + 8b503be3-6928-430c-b9ee-b53d985320a8: !Template + answer_choices: Yes ||| No + id: 8b503be3-6928-430c-b9ee-b53d985320a8 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'A/B choice 2 (bad-good, answer: B)' + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' + 9da6e7fe-1b21-411a-a8ba-1967de0c55d1: !Template + answer_choices: Yes ||| No + id: 9da6e7fe-1b21-411a-a8ba-1967de0c55d1 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences (choice order reversed) + reference: '' + b92c7d16-50fc-4274-85fd-1c3a7a938fba: !Template + answer_choices: Yes ||| No + id: b92c7d16-50fc-4274-85fd-1c3a7a938fba + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences (choice order reversed) reference: '' - 3e9d7a9e-e369-4c15-a378-18b6b2caa9b4: !Template + c47853cd-67a8-4d0c-af68-68d09334e843: !Template answer_choices: null - id: 3e9d7a9e-e369-4c15-a378-18b6b2caa9b4 + id: c47853cd-67a8-4d0c-af68-68d09334e843 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -91,11 +167,11 @@ templates: {% if shuffled_order == 0 %} - 1 + A {% else %} - 2 + B {% endif %}' metadata: !TemplateMetadata @@ -103,39 +179,14 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 - reference: '' - 96c27465-15fb-4542-ad24-960d0f7f6dc8: !Template - answer_choices: Yes ||| No - id: 96c27465-15fb-4542-ad24-960d0f7f6dc8 - jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] - }} - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences - reference: '' - b8e3fb6f-8dd4-4898-828a-3a0f8cb5f26c: !Template - answer_choices: B ||| A - id: b8e3fb6f-8dd4-4898-828a-3a0f8cb5f26c - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'B/A choice, 2 (bad-good, answer: B)' + name: random_choice_A_B_2 reference: '' - b9f81964-631c-4380-923a-2505ca8e1893: !Template + c61c531b-ad4e-49ca-a3d1-f066d4e8df15: !Template answer_choices: null - id: b9f81964-631c-4380-923a-2505ca8e1893 + id: c61c531b-ad4e-49ca-a3d1-f066d4e8df15 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -167,38 +218,37 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: random_choice_1_2_2 reference: '' - c8b8d721-878b-4e3d-acbc-ced91ccb2de5: !Template - answer_choices: null - id: c8b8d721-878b-4e3d-acbc-ced91ccb2de5 + cd9e9e21-7e96-4d47-8f5f-87183286edf1: !Template + answer_choices: A ||| B + id: cd9e9e21-7e96-4d47-8f5f-87183286edf1 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} - {% if shuffled_order == 0 %} - - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -206,30 +256,44 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: A/B choice randomized reference: '' - ea246dd2-a17a-49ba-8deb-022e460c042d: !Template - answer_choices: A ||| B - id: ea246dd2-a17a-49ba-8deb-022e460c042d - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] - }}' + ec5b64f9-1b5c-45ae-a7f3-ce42f3ca12c3: !Template + answer_choices: null + id: ec5b64f9-1b5c-45ae-a7f3-ce42f3ca12c3 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'A/B choice 1 (good-bad, answer: A)' - reference: '' - fbbd640c-dc26-44ea-8a22-312d182cd032: !Template - answer_choices: Yes ||| No - id: fbbd640c-dc26-44ea-8a22-312d182cd032 - jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] - }} - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences + name: random_choice_1_2 reference: '' diff --git a/promptsource/templates/blimp/distractor_agreement_relative_clause/templates.yaml b/promptsource/templates/blimp/distractor_agreement_relative_clause/templates.yaml index 2cd354c14..6a40bb1f8 100644 --- a/promptsource/templates/blimp/distractor_agreement_relative_clause/templates.yaml +++ b/promptsource/templates/blimp/distractor_agreement_relative_clause/templates.yaml @@ -1,24 +1,24 @@ dataset: blimp subset: distractor_agreement_relative_clause templates: - 1d3638c5-3b0b-42e0-9c1d-47f39f78deae: !Template + 1ef86170-8434-41a9-840a-1bd16b6fd17c: !Template answer_choices: null - id: 1d3638c5-3b0b-42e0-9c1d-47f39f78deae + id: 1ef86170-8434-41a9-840a-1bd16b6fd17c jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -26,11 +26,11 @@ templates: {% if shuffled_order == 0 %} - A + 1 {% else %} - B + 2 {% endif %}' metadata: !TemplateMetadata @@ -38,63 +38,58 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 - reference: '' - 4261ba02-db77-4161-b9ff-272178b64a02: !Template - answer_choices: A ||| B - id: 4261ba02-db77-4161-b9ff-272178b64a02 - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'A/B choice 2 (bad-good, answer: B)' + name: random_choice_1_2_2 reference: '' - 4409efc6-3653-4b75-8e1a-518c4443b94b: !Template + 34b4f384-755a-4cb1-b0ed-83143381d446: !Template answer_choices: Yes ||| No - id: 4409efc6-3653-4b75-8e1a-518c4443b94b - jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] - }} + id: 34b4f384-755a-4cb1-b0ed-83143381d446 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata - choices_in_prompt: false + choices_in_prompt: true metrics: - Accuracy original_task: false - name: Yes/No binary judgment for bad sentences + name: Yes/No binary judgment for good sentences reference: '' - 455e2c40-1fdc-46bf-a754-a0d066f50566: !Template + 3d9f64a4-36e7-4521-bfc9-9d51d238abfc: !Template answer_choices: Yes ||| No - id: 455e2c40-1fdc-46bf-a754-a0d066f50566 - jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] - }} + id: 3d9f64a4-36e7-4521-bfc9-9d51d238abfc + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata - choices_in_prompt: false + choices_in_prompt: true metrics: - Accuracy original_task: false - name: Yes/No binary judgment for good sentences + name: Yes/No binary judgment for bad sentences (choice order reversed) reference: '' - 7b0308ea-1e07-49ec-9aac-e28ab4ef96da: !Template + 91da17d3-ef43-4a68-9de0-729aca73b6cc: !Template answer_choices: null - id: 7b0308ea-1e07-49ec-9aac-e28ab4ef96da + id: 91da17d3-ef43-4a68-9de0-729aca73b6cc jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -102,11 +97,11 @@ templates: {% if shuffled_order == 0 %} - 1 + A {% else %} - 2 + B {% endif %}' metadata: !TemplateMetadata @@ -114,52 +109,96 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: random_choice_A_B_2 reference: '' - 90f02563-e7f5-4ebf-9476-6b7d84b240df: !Template - answer_choices: B ||| A - id: 90f02563-e7f5-4ebf-9476-6b7d84b240df - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] - }}' + a203a23d-1584-4fc9-8b30-6628050e64f6: !Template + answer_choices: Yes ||| No + id: a203a23d-1584-4fc9-8b30-6628050e64f6 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'B/A choice, 2 (bad-good, answer: B)' + original_task: false + name: Yes/No binary judgment for bad sentences reference: '' - 928cc6a0-9fa0-4fa9-baf7-b7d2ee2c453b: !Template - answer_choices: B ||| A - id: 928cc6a0-9fa0-4fa9-baf7-b7d2ee2c453b - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] - }}' + c42987e1-8f08-4e3a-b0e1-d27560a7570c: !Template + answer_choices: A ||| B + id: c42987e1-8f08-4e3a-b0e1-d27560a7570c + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'B/A choice 1 (good-bad, answer: A)' + name: A/B choice randomized (choice order reversed) reference: '' - a3c584cb-35a6-417e-92c7-25e5015f2e21: !Template + cf654a40-7389-4115-a78e-929687e209ca: !Template + answer_choices: Yes ||| No + id: cf654a40-7389-4115-a78e-929687e209ca + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences (choice order reversed) + reference: '' + e93eb5d0-f7de-4380-9ec7-4b51b09f80b7: !Template answer_choices: null - id: a3c584cb-35a6-417e-92c7-25e5015f2e21 + id: e93eb5d0-f7de-4380-9ec7-4b51b09f80b7 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -167,11 +206,11 @@ templates: {% if shuffled_order == 0 %} - 1 + A {% else %} - 2 + B {% endif %}' metadata: !TemplateMetadata @@ -179,39 +218,64 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: random_choice_A_B reference: '' - a66605d0-f318-4a5f-bba7-d2df67f01d6d: !Template + f2c44dce-ea0b-4736-b602-7edf37bafc7a: !Template answer_choices: A ||| B - id: a66605d0-f318-4a5f-bba7-d2df67f01d6d - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] - }}' + id: f2c44dce-ea0b-4736-b602-7edf37bafc7a + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'A/B choice 1 (good-bad, answer: A)' + name: A/B choice randomized reference: '' - cf595472-4ecf-4147-b76d-cb47fc879884: !Template + f5301a1c-4aea-42d2-a36a-1dc04c89733a: !Template answer_choices: null - id: cf595472-4ecf-4147-b76d-cb47fc879884 + id: f5301a1c-4aea-42d2-a36a-1dc04c89733a jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -219,11 +283,11 @@ templates: {% if shuffled_order == 0 %} - A + 1 {% else %} - B + 2 {% endif %}' metadata: !TemplateMetadata @@ -231,5 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: random_choice_1_2 reference: '' diff --git a/promptsource/templates/blimp/drop_argument/templates.yaml b/promptsource/templates/blimp/drop_argument/templates.yaml index 66035c31b..7a8c7d4a7 100644 --- a/promptsource/templates/blimp/drop_argument/templates.yaml +++ b/promptsource/templates/blimp/drop_argument/templates.yaml @@ -1,74 +1,56 @@ dataset: blimp subset: drop_argument templates: - 347ccb81-2e47-4cae-8d0a-736888d89ff8: !Template - answer_choices: A ||| B - id: 347ccb81-2e47-4cae-8d0a-736888d89ff8 - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'A/B choice 1 (good-bad, answer: A)' - reference: '' - 3f3c1792-3a68-475d-a817-5219773dae8d: !Template - answer_choices: B ||| A - id: 3f3c1792-3a68-475d-a817-5219773dae8d - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'B/A choice 1 (good-bad, answer: A)' - reference: '' - 6501fec5-fb69-414a-9eee-75587833f1a0: !Template + 0bf1ba17-165d-40f0-9b9a-05d585d8809d: !Template answer_choices: Yes ||| No - id: 6501fec5-fb69-414a-9eee-75587833f1a0 - jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] - }} + id: 0bf1ba17-165d-40f0-9b9a-05d585d8809d + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata - choices_in_prompt: false + choices_in_prompt: true metrics: - Accuracy original_task: false - name: Yes/No binary judgment for good sentences + name: Yes/No binary judgment for bad sentences reference: '' - 72e37f51-2c07-43b6-a5bc-305e89b58d9c: !Template + 1870d493-7858-454d-8bf2-5991584b3b5b: !Template answer_choices: Yes ||| No - id: 72e37f51-2c07-43b6-a5bc-305e89b58d9c - jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] - }} + id: 1870d493-7858-454d-8bf2-5991584b3b5b + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata - choices_in_prompt: false + choices_in_prompt: true metrics: - Accuracy original_task: false - name: Yes/No binary judgment for bad sentences + name: Yes/No binary judgment for bad sentences (choice order reversed) reference: '' - 75f965d1-8cc7-49d0-b496-7eee15116642: !Template + 787990bf-9ad9-4855-91e8-58de425ed0dc: !Template answer_choices: null - id: 75f965d1-8cc7-49d0-b496-7eee15116642 + id: 787990bf-9ad9-4855-91e8-58de425ed0dc jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -76,11 +58,11 @@ templates: {% if shuffled_order == 0 %} - A + 1 {% else %} - B + 2 {% endif %}' metadata: !TemplateMetadata @@ -88,11 +70,27 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: random_choice_1_2 reference: '' - a5553fc9-c58a-4a8f-adb4-35ed4a1510a0: !Template + 83e5275e-eec3-473a-a661-86de738ce309: !Template + answer_choices: Yes ||| No + id: 83e5275e-eec3-473a-a661-86de738ce309 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences (choice order reversed) + reference: '' + a40019ff-ec9a-4bf9-9939-6843907680ea: !Template answer_choices: null - id: a5553fc9-c58a-4a8f-adb4-35ed4a1510a0 + id: a40019ff-ec9a-4bf9-9939-6843907680ea jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -129,9 +127,9 @@ templates: original_task: true name: random_choice_1_2_2 reference: '' - b7709a79-a79e-42d6-b34c-816d5ee4221c: !Template + a8f2fb21-8838-4405-a871-4de7fd62aa47: !Template answer_choices: null - id: b7709a79-a79e-42d6-b34c-816d5ee4221c + id: a8f2fb21-8838-4405-a871-4de7fd62aa47 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer A or B. @@ -168,37 +166,78 @@ templates: original_task: true name: random_choice_A_B_2 reference: '' - cffaca3b-3201-413c-9eaf-23a19714ef69: !Template + d02f9f83-bfcd-41fa-b4d8-97781ce9561b: !Template answer_choices: A ||| B - id: cffaca3b-3201-413c-9eaf-23a19714ef69 - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] - }}' + id: d02f9f83-bfcd-41fa-b4d8-97781ce9561b + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'A/B choice 2 (bad-good, answer: B)' + name: A/B choice randomized (choice order reversed) reference: '' - e859c136-33c7-426e-bb67-a48481d452ce: !Template + d981d4f8-78c9-4891-b54c-49f28a4142d6: !Template + answer_choices: Yes ||| No + id: d981d4f8-78c9-4891-b54c-49f28a4142d6 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' + dd8d9afc-4217-4960-babe-e281776fe11b: !Template answer_choices: null - id: e859c136-33c7-426e-bb67-a48481d452ce + id: dd8d9afc-4217-4960-babe-e281776fe11b jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -206,11 +245,11 @@ templates: {% if shuffled_order == 0 %} - 1 + A {% else %} - 2 + B {% endif %}' metadata: !TemplateMetadata @@ -218,18 +257,43 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: random_choice_A_B reference: '' - fc90633d-b1ad-43e8-9874-223f5d59ac0a: !Template - answer_choices: B ||| A - id: fc90633d-b1ad-43e8-9874-223f5d59ac0a - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] - }}' + df229dea-02cd-4d69-85d7-386de829b690: !Template + answer_choices: A ||| B + id: df229dea-02cd-4d69-85d7-386de829b690 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'B/A choice, 2 (bad-good, answer: B)' + name: A/B choice randomized reference: '' diff --git a/promptsource/templates/blimp/ellipsis_n_bar_1/templates.yaml b/promptsource/templates/blimp/ellipsis_n_bar_1/templates.yaml index 7525b87bd..a29c42aa1 100644 --- a/promptsource/templates/blimp/ellipsis_n_bar_1/templates.yaml +++ b/promptsource/templates/blimp/ellipsis_n_bar_1/templates.yaml @@ -1,24 +1,12 @@ dataset: blimp subset: ellipsis_n_bar_1 templates: - 248f9e3f-9bdb-456b-add3-31b9b18db5fa: !Template - answer_choices: Yes ||| No - id: 248f9e3f-9bdb-456b-add3-31b9b18db5fa - jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] - }} - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences - reference: '' - 327a1cc7-80d0-4a99-a0c9-0f7207beb81c: !Template + 01a44c1f-0795-46f2-bcbe-fe0c4d913640: !Template answer_choices: null - id: 327a1cc7-80d0-4a99-a0c9-0f7207beb81c + id: 01a44c1f-0795-46f2-bcbe-fe0c4d913640 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -50,39 +38,42 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: random_choice_A_B reference: '' - 8b6db999-81c8-4c88-ae9a-de07cd958b6f: !Template - answer_choices: B ||| A - id: 8b6db999-81c8-4c88-ae9a-de07cd958b6f - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] - }}' + 3eac83e5-11ca-46d4-9b0e-0516b6088971: !Template + answer_choices: Yes ||| No + id: 3eac83e5-11ca-46d4-9b0e-0516b6088971 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'B/A choice 1 (good-bad, answer: A)' + original_task: false + name: Yes/No binary judgment for good sentences reference: '' - 9a7f3418-0e1b-4981-8059-83981dcc4ab4: !Template + 8f6d4252-e867-4e73-884e-e68ff6ea3b07: !Template answer_choices: null - id: 9a7f3418-0e1b-4981-8059-83981dcc4ab4 + id: 8f6d4252-e867-4e73-884e-e68ff6ea3b07 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -90,11 +81,11 @@ templates: {% if shuffled_order == 0 %} - 1 + A {% else %} - 2 + B {% endif %}' metadata: !TemplateMetadata @@ -102,51 +93,96 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: random_choice_A_B_2 reference: '' - cce1cedf-4fa6-4a7d-8b07-07e845ee8d18: !Template + 9c2181f4-f828-4486-9bc6-66b474a5b8b1: !Template answer_choices: A ||| B - id: cce1cedf-4fa6-4a7d-8b07-07e845ee8d18 - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] - }}' + id: 9c2181f4-f828-4486-9bc6-66b474a5b8b1 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'A/B choice 1 (good-bad, answer: A)' + name: A/B choice randomized reference: '' - e0e0b64f-851b-495a-9716-91e950479bc1: !Template + a257c15a-e3a3-4362-a3b4-44623a93e608: !Template answer_choices: Yes ||| No - id: e0e0b64f-851b-495a-9716-91e950479bc1 - jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] - }} + id: a257c15a-e3a3-4362-a3b4-44623a93e608 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata - choices_in_prompt: false + choices_in_prompt: true metrics: - Accuracy original_task: false - name: Yes/No binary judgment for good sentences + name: Yes/No binary judgment for bad sentences (choice order reversed) + reference: '' + ce12d33f-ee98-49b8-825b-0f0b98ddde80: !Template + answer_choices: Yes ||| No + id: ce12d33f-ee98-49b8-825b-0f0b98ddde80 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences reference: '' - e9b63520-d889-474a-92f8-fdde852b1af6: !Template + da36c020-4ca0-40be-aa54-96628a9e1903: !Template answer_choices: null - id: e9b63520-d889-474a-92f8-fdde852b1af6 + id: da36c020-4ca0-40be-aa54-96628a9e1903 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -154,11 +190,11 @@ templates: {% if shuffled_order == 0 %} - A + 1 {% else %} - B + 2 {% endif %}' metadata: !TemplateMetadata @@ -166,37 +202,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B - reference: '' - f80d1616-daec-4448-b30c-9ae3251965ad: !Template - answer_choices: B ||| A - id: f80d1616-daec-4448-b30c-9ae3251965ad - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'B/A choice, 2 (bad-good, answer: B)' - reference: '' - ff8f539f-13f6-45a6-ba3b-6ce5ecdf1947: !Template - answer_choices: A ||| B - id: ff8f539f-13f6-45a6-ba3b-6ce5ecdf1947 - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'A/B choice 2 (bad-good, answer: B)' + name: random_choice_1_2 reference: '' - ffa392d4-c1b6-45d7-95d8-5a0f67fbafcd: !Template + e07ae199-198b-4bae-9405-2a306c0cebe1: !Template answer_choices: null - id: ffa392d4-c1b6-45d7-95d8-5a0f67fbafcd + id: e07ae199-198b-4bae-9405-2a306c0cebe1 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -233,3 +243,57 @@ templates: original_task: true name: random_choice_1_2_2 reference: '' + e8f7cb79-ec4c-466d-a332-295727f12985: !Template + answer_choices: Yes ||| No + id: e8f7cb79-ec4c-466d-a332-295727f12985 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences (choice order reversed) + reference: '' + efc5e0a3-5b64-46da-afb6-1f71208300e7: !Template + answer_choices: A ||| B + id: efc5e0a3-5b64-46da-afb6-1f71208300e7 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: A/B choice randomized (choice order reversed) + reference: '' diff --git a/promptsource/templates/blimp/ellipsis_n_bar_2/templates.yaml b/promptsource/templates/blimp/ellipsis_n_bar_2/templates.yaml index b790320c4..3ca9f67cc 100644 --- a/promptsource/templates/blimp/ellipsis_n_bar_2/templates.yaml +++ b/promptsource/templates/blimp/ellipsis_n_bar_2/templates.yaml @@ -1,37 +1,40 @@ dataset: blimp subset: ellipsis_n_bar_2 templates: - 042361b3-06b8-4ae9-a3e8-1f14b7be1306: !Template - answer_choices: A ||| B - id: 042361b3-06b8-4ae9-a3e8-1f14b7be1306 - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] - }}' + 057469f7-603d-44f3-ae1e-1bc6e8629fdd: !Template + answer_choices: Yes ||| No + id: 057469f7-603d-44f3-ae1e-1bc6e8629fdd + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'A/B choice 2 (bad-good, answer: B)' + original_task: false + name: Yes/No binary judgment for good sentences (choice order reversed) reference: '' - 06940021-1f86-42f2-bfeb-f4c019a21578: !Template + 2ee1824b-0c3c-43c3-b401-6ec17d7b539f: !Template answer_choices: null - id: 06940021-1f86-42f2-bfeb-f4c019a21578 + id: 2ee1824b-0c3c-43c3-b401-6ec17d7b539f jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -39,11 +42,11 @@ templates: {% if shuffled_order == 0 %} - A + 1 {% else %} - B + 2 {% endif %}' metadata: !TemplateMetadata @@ -51,76 +54,53 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 - reference: '' - 13411dd1-e05f-464c-b8f1-40dd3d533ac6: !Template - answer_choices: B ||| A - id: 13411dd1-e05f-464c-b8f1-40dd3d533ac6 - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'B/A choice, 2 (bad-good, answer: B)' + name: random_choice_1_2 reference: '' - 25102e92-f738-455b-af3d-ee798b1ddfcd: !Template + 31fd3e98-fe31-4faf-a773-ff89327a6be9: !Template answer_choices: Yes ||| No - id: 25102e92-f738-455b-af3d-ee798b1ddfcd - jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] - }} - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences - reference: '' - 31891c60-4c57-4c5d-9146-77c8795c2d84: !Template - answer_choices: B ||| A - id: 31891c60-4c57-4c5d-9146-77c8795c2d84 - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] - }}' + id: 31fd3e98-fe31-4faf-a773-ff89327a6be9 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'B/A choice 1 (good-bad, answer: A)' + original_task: false + name: Yes/No binary judgment for bad sentences (choice order reversed) reference: '' - 71113537-7e9b-4f83-a5b2-d1f6ea8b07e8: !Template - answer_choices: null - id: 71113537-7e9b-4f83-a5b2-d1f6ea8b07e8 + 3ce21da7-d059-465d-99b7-202e9ab53141: !Template + answer_choices: A ||| B + id: 3ce21da7-d059-465d-99b7-202e9ab53141 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -128,14 +108,14 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: A/B choice randomized reference: '' - 812791bf-dad4-481c-8dca-1fff8626799b: !Template + 61cdb2a7-8710-43de-a1a9-4d537edda041: !Template answer_choices: null - id: 812791bf-dad4-481c-8dca-1fff8626799b + id: 61cdb2a7-8710-43de-a1a9-4d537edda041 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -167,26 +147,52 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: random_choice_A_B_2 reference: '' - 990f63ec-4e22-40e9-8178-e58454dc3ede: !Template - answer_choices: Yes ||| No - id: 990f63ec-4e22-40e9-8178-e58454dc3ede - jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] - }} + 7613236f-2c75-4bfd-8f24-e9a85e21d123: !Template + answer_choices: A ||| B + id: 7613236f-2c75-4bfd-8f24-e9a85e21d123 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata - choices_in_prompt: false + choices_in_prompt: true metrics: - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences + original_task: true + name: A/B choice randomized (choice order reversed) reference: '' - a11f17a4-6e2d-4b5d-9833-b73dcd987702: !Template + 93ae78e3-c813-4286-b3e7-fea762ccc90b: !Template answer_choices: null - id: a11f17a4-6e2d-4b5d-9833-b73dcd987702 + id: 93ae78e3-c813-4286-b3e7-fea762ccc90b jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -218,18 +224,76 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: random_choice_1_2_2 reference: '' - b87e7cc8-ce0a-4dee-a256-05d61fa2efbb: !Template - answer_choices: A ||| B - id: b87e7cc8-ce0a-4dee-a256-05d61fa2efbb - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] - }}' + 9f359e11-ac9a-4132-bfec-627657e99d98: !Template + answer_choices: Yes ||| No + id: 9f359e11-ac9a-4132-bfec-627657e99d98 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' + a37850c7-1d04-4644-8400-cdcc2fa78ada: !Template + answer_choices: Yes ||| No + id: a37850c7-1d04-4644-8400-cdcc2fa78ada + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' + f8fd5129-ef43-4859-9e4a-6b52f3f98368: !Template + answer_choices: null + id: f8fd5129-ef43-4859-9e4a-6b52f3f98368 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'A/B choice 1 (good-bad, answer: A)' + name: random_choice_A_B reference: '' diff --git a/promptsource/templates/blimp/existential_there_object_raising/templates.yaml b/promptsource/templates/blimp/existential_there_object_raising/templates.yaml index e4dd329ae..6c9c9f7b6 100644 --- a/promptsource/templates/blimp/existential_there_object_raising/templates.yaml +++ b/promptsource/templates/blimp/existential_there_object_raising/templates.yaml @@ -1,36 +1,12 @@ dataset: blimp subset: existential_there_object_raising templates: - 01e4ede6-3cf1-443b-ad82-69b7f67112ac: !Template - answer_choices: Yes ||| No - id: 01e4ede6-3cf1-443b-ad82-69b7f67112ac - jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] - }} - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences - reference: '' - 278e0258-a74e-43c0-8b6e-c1cda8c298a7: !Template - answer_choices: Yes ||| No - id: 278e0258-a74e-43c0-8b6e-c1cda8c298a7 - jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] - }} - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences - reference: '' - 4c318fc5-fd44-4fda-9dac-24eb65793fcf: !Template + 09d49336-b67a-4fd8-a1f2-227cfa11f348: !Template answer_choices: null - id: 4c318fc5-fd44-4fda-9dac-24eb65793fcf + id: 09d49336-b67a-4fd8-a1f2-227cfa11f348 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -62,38 +38,37 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: random_choice_A_B_2 reference: '' - 4eb61e9c-417f-48f4-aae8-bb7f82e02f8d: !Template - answer_choices: null - id: 4eb61e9c-417f-48f4-aae8-bb7f82e02f8d + 363802c7-c284-44d2-ad3a-5f82a3f3aca2: !Template + answer_choices: A ||| B + id: 363802c7-c284-44d2-ad3a-5f82a3f3aca2 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -101,27 +76,68 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: A/B choice randomized (choice order reversed) reference: '' - 7b0dc8bf-eba0-4c7a-b7f0-29b276f36ae6: !Template + 5691f73d-2d67-49f2-ae26-fe93a173d89b: !Template answer_choices: A ||| B - id: 7b0dc8bf-eba0-4c7a-b7f0-29b276f36ae6 - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] - }}' + id: 5691f73d-2d67-49f2-ae26-fe93a173d89b + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'A/B choice 1 (good-bad, answer: A)' + name: A/B choice randomized + reference: '' + 647bbd76-ae39-44d8-9240-6e4a4ce69783: !Template + answer_choices: Yes ||| No + id: 647bbd76-ae39-44d8-9240-6e4a4ce69783 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences reference: '' - 7b2e11e9-0417-4c96-8b67-dde9a884ca5d: !Template + 8665f45a-d651-495e-b947-70795e79952e: !Template answer_choices: null - id: 7b2e11e9-0417-4c96-8b67-dde9a884ca5d + id: 8665f45a-d651-495e-b947-70795e79952e jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -153,65 +169,113 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: random_choice_1_2 + reference: '' + a2ffe96e-ca1c-47f0-93d5-d84cca7f3bb7: !Template + answer_choices: Yes ||| No + id: a2ffe96e-ca1c-47f0-93d5-d84cca7f3bb7 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences (choice order reversed) reference: '' - 9dd2b833-7a83-4024-8371-b4abcc3744b1: !Template - answer_choices: B ||| A - id: 9dd2b833-7a83-4024-8371-b4abcc3744b1 - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] - }}' + af85dede-9050-430c-9993-5bb4a90c94b8: !Template + answer_choices: null + id: af85dede-9050-430c-9993-5bb4a90c94b8 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'B/A choice, 2 (bad-good, answer: B)' + name: random_choice_1_2_2 reference: '' - 9f1529b8-a026-46b4-88a4-0dc78b2c1873: !Template - answer_choices: A ||| B - id: 9f1529b8-a026-46b4-88a4-0dc78b2c1873 - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] - }}' + b37c08ad-edac-4599-bd7f-38adc42a5fd4: !Template + answer_choices: Yes ||| No + id: b37c08ad-edac-4599-bd7f-38adc42a5fd4 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'A/B choice 2 (bad-good, answer: B)' + original_task: false + name: Yes/No binary judgment for good sentences (choice order reversed) reference: '' - a6fca9f8-42cb-4ff4-b264-bc6171793913: !Template - answer_choices: B ||| A - id: a6fca9f8-42cb-4ff4-b264-bc6171793913 - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] - }}' + cf7da09c-ad33-4adc-b210-beca6b849501: !Template + answer_choices: Yes ||| No + id: cf7da09c-ad33-4adc-b210-beca6b849501 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'B/A choice 1 (good-bad, answer: A)' + original_task: false + name: Yes/No binary judgment for good sentences reference: '' - d43034bb-0349-462e-8b96-077297c7221e: !Template + d2f71332-89e9-47b9-89a6-32b651accfec: !Template answer_choices: null - id: d43034bb-0349-462e-8b96-077297c7221e + id: d2f71332-89e9-47b9-89a6-32b651accfec jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -219,11 +283,11 @@ templates: {% if shuffled_order == 0 %} - 1 + A {% else %} - 2 + B {% endif %}' metadata: !TemplateMetadata @@ -231,5 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: random_choice_A_B reference: '' diff --git a/promptsource/templates/blimp/existential_there_quantifiers_1/templates.yaml b/promptsource/templates/blimp/existential_there_quantifiers_1/templates.yaml index 04e7b9f4c..338829700 100644 --- a/promptsource/templates/blimp/existential_there_quantifiers_1/templates.yaml +++ b/promptsource/templates/blimp/existential_there_quantifiers_1/templates.yaml @@ -1,24 +1,12 @@ dataset: blimp subset: existential_there_quantifiers_1 templates: - 00750be4-7bbf-4358-b7e1-d601a6f3a545: !Template - answer_choices: Yes ||| No - id: 00750be4-7bbf-4358-b7e1-d601a6f3a545 - jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] - }} - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences - reference: '' - 6c44cd14-a0cb-47ec-b274-5b69db0f8291: !Template + 0ea04c16-ffaa-456a-8829-db4bb63e6d2b: !Template answer_choices: null - id: 6c44cd14-a0cb-47ec-b274-5b69db0f8291 + id: 0ea04c16-ffaa-456a-8829-db4bb63e6d2b jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -50,51 +38,53 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: random_choice_A_B reference: '' - 6e7cf462-c2ff-49c6-b986-0ba9003be9cc: !Template - answer_choices: B ||| A - id: 6e7cf462-c2ff-49c6-b986-0ba9003be9cc - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] - }}' + 2c4d2d8d-d44a-4117-a1d5-06df8e6b19f5: !Template + answer_choices: Yes ||| No + id: 2c4d2d8d-d44a-4117-a1d5-06df8e6b19f5 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'B/A choice, 2 (bad-good, answer: B)' + original_task: false + name: Yes/No binary judgment for good sentences reference: '' - 80abf7a6-8b8c-4a15-956c-5bb3aee4b84e: !Template - answer_choices: null - id: 80abf7a6-8b8c-4a15-956c-5bb3aee4b84e + 3bccd33b-c9b6-4a6e-886c-764c23387bda: !Template + answer_choices: A ||| B + id: 3bccd33b-c9b6-4a6e-886c-764c23387bda jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} - {% if shuffled_order == 0 %} - - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -102,14 +92,30 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: A/B choice randomized reference: '' - 9fbb74c6-7d48-40f7-9262-ec05ca9ea1e0: !Template + 4fe1f1c8-fadd-4281-980d-bf2e7643e2b5: !Template + answer_choices: Yes ||| No + id: 4fe1f1c8-fadd-4281-980d-bf2e7643e2b5 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' + 580ef5e1-c4a5-41f5-a76d-c063aed4ab00: !Template answer_choices: null - id: 9fbb74c6-7d48-40f7-9262-ec05ca9ea1e0 + id: 580ef5e1-c4a5-41f5-a76d-c063aed4ab00 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -141,77 +147,135 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: random_choice_1_2 reference: '' - b4e82d40-7e52-4118-ad35-c2022b8faec7: !Template - answer_choices: B ||| A - id: b4e82d40-7e52-4118-ad35-c2022b8faec7 - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] - }}' + 617cf5f3-4ae5-461b-af29-cb2a124cba1e: !Template + answer_choices: Yes ||| No + id: 617cf5f3-4ae5-461b-af29-cb2a124cba1e + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'B/A choice 1 (good-bad, answer: A)' + original_task: false + name: Yes/No binary judgment for bad sentences (choice order reversed) reference: '' - bede45a1-5834-4444-8c52-986f3c273a04: !Template - answer_choices: Yes ||| No - id: bede45a1-5834-4444-8c52-986f3c273a04 - jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] - }} + 89eb62ff-6431-4bab-974c-344e03ddd222: !Template + answer_choices: null + id: 89eb62ff-6431-4bab-974c-344e03ddd222 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' metadata: !TemplateMetadata - choices_in_prompt: false + choices_in_prompt: true metrics: - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences + original_task: true + name: random_choice_A_B_2 reference: '' - e1f0143e-ac66-4187-bf06-50835d581592: !Template + 93993a21-914d-4a24-80dd-a76fdb1e0884: !Template answer_choices: A ||| B - id: e1f0143e-ac66-4187-bf06-50835d581592 - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] - }}' + id: 93993a21-914d-4a24-80dd-a76fdb1e0884 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'A/B choice 2 (bad-good, answer: B)' + name: A/B choice randomized (choice order reversed) reference: '' - f0785228-7647-450a-b6b8-11e7f9120bbe: !Template - answer_choices: A ||| B - id: f0785228-7647-450a-b6b8-11e7f9120bbe - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] - }}' + 9af98d9b-17f6-4d8c-8567-2d7eb64037cd: !Template + answer_choices: Yes ||| No + id: 9af98d9b-17f6-4d8c-8567-2d7eb64037cd + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'A/B choice 1 (good-bad, answer: A)' + original_task: false + name: Yes/No binary judgment for good sentences (choice order reversed) reference: '' - f373c4a6-1782-4a00-b513-c960f2d57dfc: !Template + a6e4de47-8868-48a6-beb9-72ea62ca9286: !Template answer_choices: null - id: f373c4a6-1782-4a00-b513-c960f2d57dfc + id: a6e4de47-8868-48a6-beb9-72ea62ca9286 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -219,11 +283,11 @@ templates: {% if shuffled_order == 0 %} - A + 1 {% else %} - B + 2 {% endif %}' metadata: !TemplateMetadata @@ -231,5 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: random_choice_1_2_2 reference: '' diff --git a/promptsource/templates/blimp/existential_there_quantifiers_2/templates.yaml b/promptsource/templates/blimp/existential_there_quantifiers_2/templates.yaml index d34d97e23..be905507e 100644 --- a/promptsource/templates/blimp/existential_there_quantifiers_2/templates.yaml +++ b/promptsource/templates/blimp/existential_there_quantifiers_2/templates.yaml @@ -1,35 +1,9 @@ dataset: blimp subset: existential_there_quantifiers_2 templates: - 105555b0-4d08-4590-94e7-12ad3343ea1e: !Template - answer_choices: A ||| B - id: 105555b0-4d08-4590-94e7-12ad3343ea1e - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'A/B choice 1 (good-bad, answer: A)' - reference: '' - 1fcb97d3-5ed3-4c61-9421-43354409949e: !Template - answer_choices: B ||| A - id: 1fcb97d3-5ed3-4c61-9421-43354409949e - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'B/A choice, 2 (bad-good, answer: B)' - reference: '' - 5ca43b41-0672-4b19-8808-26fe4e21b307: !Template + 1f4dc210-9f43-417b-b9d5-3d0c320513f9: !Template answer_choices: null - id: 5ca43b41-0672-4b19-8808-26fe4e21b307 + id: 1f4dc210-9f43-417b-b9d5-3d0c320513f9 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -66,12 +40,12 @@ templates: original_task: true name: random_choice_1_2_2 reference: '' - 69ebf4bc-ca4a-4e0b-b448-4b90baba3d72: !Template + 5aa1a0d9-9e17-4cf9-8e2d-d3b6eaf28d6b: !Template answer_choices: null - id: 69ebf4bc-ca4a-4e0b-b448-4b90baba3d72 + id: 5aa1a0d9-9e17-4cf9-8e2d-d3b6eaf28d6b jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -103,11 +77,103 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: random_choice_A_B + reference: '' + 69ab9d17-de35-456e-a109-62a2ebb018db: !Template + answer_choices: A ||| B + id: 69ab9d17-de35-456e-a109-62a2ebb018db + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: A/B choice randomized (choice order reversed) + reference: '' + 76d6c700-febc-4dad-91f3-536a74b4996d: !Template + answer_choices: A ||| B + id: 76d6c700-febc-4dad-91f3-536a74b4996d + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: A/B choice randomized reference: '' - d2f78114-9249-4777-bfef-fb3c457ea14b: !Template + 885625f9-0946-4b4a-b024-781259ff2540: !Template + answer_choices: Yes ||| No + id: 885625f9-0946-4b4a-b024-781259ff2540 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences (choice order reversed) + reference: '' + 9b3d0cc7-0a21-4267-b924-7b9fcb54e7dd: !Template answer_choices: null - id: d2f78114-9249-4777-bfef-fb3c457ea14b + id: 9b3d0cc7-0a21-4267-b924-7b9fcb54e7dd jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer 1 or 2. @@ -144,24 +210,28 @@ templates: original_task: true name: random_choice_1_2 reference: '' - dd960fc2-ff85-4d96-893c-e9cc37d95b61: !Template + b4fffc65-d9d8-41c8-a1cf-ff08de5eccc5: !Template answer_choices: Yes ||| No - id: dd960fc2-ff85-4d96-893c-e9cc37d95b61 - jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] - }} + id: b4fffc65-d9d8-41c8-a1cf-ff08de5eccc5 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata - choices_in_prompt: false + choices_in_prompt: true metrics: - Accuracy original_task: false name: Yes/No binary judgment for bad sentences reference: '' - f033067b-e2bc-4e53-b5f4-e614767a6c13: !Template + bf7e7a58-f63e-4435-b544-6ce48e58e015: !Template answer_choices: null - id: f033067b-e2bc-4e53-b5f4-e614767a6c13 + id: bf7e7a58-f63e-4435-b544-6ce48e58e015 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -193,43 +263,37 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: random_choice_A_B_2 reference: '' - f1405ef4-a250-4981-9d68-3dc2d2d0e583: !Template + c4abd7d4-057a-41de-9c6e-7bf05e6fe197: !Template answer_choices: Yes ||| No - id: f1405ef4-a250-4981-9d68-3dc2d2d0e583 - jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] - }} - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences - reference: '' - fc212969-d0cf-4942-aaea-75b55d3f453f: !Template - answer_choices: B ||| A - id: fc212969-d0cf-4942-aaea-75b55d3f453f - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] - }}' + id: c4abd7d4-057a-41de-9c6e-7bf05e6fe197 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'B/A choice 1 (good-bad, answer: A)' + original_task: false + name: Yes/No binary judgment for bad sentences (choice order reversed) reference: '' - fe547b2b-12f2-4fef-a3f6-c2e45dce6309: !Template - answer_choices: A ||| B - id: fe547b2b-12f2-4fef-a3f6-c2e45dce6309 - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] - }}' + d6896ade-c253-4291-b946-ed3b815c531e: !Template + answer_choices: Yes ||| No + id: d6896ade-c253-4291-b946-ed3b815c531e + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'A/B choice 2 (bad-good, answer: B)' + original_task: false + name: Yes/No binary judgment for good sentences reference: '' diff --git a/promptsource/templates/blimp/existential_there_subject_raising/templates.yaml b/promptsource/templates/blimp/existential_there_subject_raising/templates.yaml index c845a9cbc..3fda7566a 100644 --- a/promptsource/templates/blimp/existential_there_subject_raising/templates.yaml +++ b/promptsource/templates/blimp/existential_there_subject_raising/templates.yaml @@ -1,47 +1,103 @@ dataset: blimp subset: existential_there_subject_raising templates: - 003be84d-8ca5-4e8f-8507-6eedee51025c: !Template - answer_choices: B ||| A - id: 003be84d-8ca5-4e8f-8507-6eedee51025c - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] - }}' + 01383138-9dc9-4aec-9d30-29a3bc45396a: !Template + answer_choices: null + id: 01383138-9dc9-4aec-9d30-29a3bc45396a + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'B/A choice, 2 (bad-good, answer: B)' + name: random_choice_A_B reference: '' - 3e7e588f-c8a9-4eff-aa3b-9cec2dd1b82c: !Template - answer_choices: Yes ||| No - id: 3e7e588f-c8a9-4eff-aa3b-9cec2dd1b82c - jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] - }} + 333bed79-11a0-48f8-8c4c-d920082ce7c8: !Template + answer_choices: null + id: 333bed79-11a0-48f8-8c4c-d920082ce7c8 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' metadata: !TemplateMetadata - choices_in_prompt: false + choices_in_prompt: true metrics: - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences + original_task: true + name: random_choice_1_2_2 reference: '' - 41382acf-6f8d-4835-aee7-7dbe0d5d1c7b: !Template - answer_choices: A ||| B - id: 41382acf-6f8d-4835-aee7-7dbe0d5d1c7b - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] - }}' + 6311612b-fccf-4e5b-bc63-f674f02134ce: !Template + answer_choices: Yes ||| No + id: 6311612b-fccf-4e5b-bc63-f674f02134ce + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'A/B choice 1 (good-bad, answer: A)' + original_task: false + name: Yes/No binary judgment for good sentences reference: '' - 41ae3a18-ed4e-4cd1-8dc5-ef844b5f5c1d: !Template + 76c481ad-4017-48cd-9b51-9285c4c485d9: !Template answer_choices: null - id: 41ae3a18-ed4e-4cd1-8dc5-ef844b5f5c1d + id: 76c481ad-4017-48cd-9b51-9285c4c485d9 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer A or B. @@ -78,38 +134,28 @@ templates: original_task: true name: random_choice_A_B_2 reference: '' - 538bbe9b-e66c-4fb9-b69f-d15b9925e671: !Template - answer_choices: B ||| A - id: 538bbe9b-e66c-4fb9-b69f-d15b9925e671 - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'B/A choice 1 (good-bad, answer: A)' - reference: '' - 68a68f4e-6347-4dfa-a8fb-06e9a3a42eaf: !Template - answer_choices: A ||| B - id: 68a68f4e-6347-4dfa-a8fb-06e9a3a42eaf - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] - }}' + 7d67cd77-1744-4ab2-bd17-5e45a4ebe7ab: !Template + answer_choices: Yes ||| No + id: 7d67cd77-1744-4ab2-bd17-5e45a4ebe7ab + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'A/B choice 2 (bad-good, answer: B)' + original_task: false + name: Yes/No binary judgment for good sentences (choice order reversed) reference: '' - 6e252204-c288-4377-94cc-eba6c2238ce6: !Template + 838ab848-4ec8-4dda-bcbc-ed8018ae1dce: !Template answer_choices: null - id: 6e252204-c288-4377-94cc-eba6c2238ce6 + id: 838ab848-4ec8-4dda-bcbc-ed8018ae1dce jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -141,38 +187,69 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: random_choice_1_2 reference: '' - 7295849e-7a4c-45e4-a551-64ee347a51c3: !Template - answer_choices: null - id: 7295849e-7a4c-45e4-a551-64ee347a51c3 + 9757090f-3401-40d3-921f-b5a686b23423: !Template + answer_choices: Yes ||| No + id: 9757090f-3401-40d3-921f-b5a686b23423 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences (choice order reversed) + reference: '' + a046e56c-aec4-4d20-abc9-bd519221ba13: !Template + answer_choices: Yes ||| No + id: a046e56c-aec4-4d20-abc9-bd519221ba13 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' + b5f72555-cbba-4156-b4dd-4c071ddabd34: !Template + answer_choices: A ||| B + id: b5f72555-cbba-4156-b4dd-4c071ddabd34 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} - {% if shuffled_order == 0 %} - - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -180,38 +257,37 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: A/B choice randomized (choice order reversed) reference: '' - 7977d568-25a4-4fb6-9486-02e851bbbeae: !Template - answer_choices: null - id: 7977d568-25a4-4fb6-9486-02e851bbbeae + d3055b67-c20e-4354-8f2c-bcb7da50775a: !Template + answer_choices: A ||| B + id: d3055b67-c20e-4354-8f2c-bcb7da50775a jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} - {% if shuffled_order == 0 %} - - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -219,17 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B - reference: '' - 8823b041-ea11-45a6-b1ff-f3ca2df932f9: !Template - answer_choices: Yes ||| No - id: 8823b041-ea11-45a6-b1ff-f3ca2df932f9 - jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] - }} - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences + name: A/B choice randomized reference: '' diff --git a/promptsource/templates/blimp/expletive_it_object_raising/templates.yaml b/promptsource/templates/blimp/expletive_it_object_raising/templates.yaml index 0bf305cc3..a7c3486d5 100644 --- a/promptsource/templates/blimp/expletive_it_object_raising/templates.yaml +++ b/promptsource/templates/blimp/expletive_it_object_raising/templates.yaml @@ -1,50 +1,82 @@ dataset: blimp subset: expletive_it_object_raising templates: - 2488fd05-627b-4a63-947d-3853910430b3: !Template - answer_choices: B ||| A - id: 2488fd05-627b-4a63-947d-3853910430b3 - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] - }}' + 6dbec4b3-8a72-4f82-b6b5-1b5dbd5af858: !Template + answer_choices: Yes ||| No + id: 6dbec4b3-8a72-4f82-b6b5-1b5dbd5af858 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'B/A choice 1 (good-bad, answer: A)' + original_task: false + name: Yes/No binary judgment for good sentences reference: '' - 296eb36b-7f17-4def-9c55-b23aa400dfee: !Template + 6eebdcd0-e41a-4370-96bc-53af32d41ba8: !Template answer_choices: Yes ||| No - id: 296eb36b-7f17-4def-9c55-b23aa400dfee - jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] - }} + id: 6eebdcd0-e41a-4370-96bc-53af32d41ba8 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata - choices_in_prompt: false + choices_in_prompt: true metrics: - Accuracy original_task: false - name: Yes/No binary judgment for bad sentences + name: Yes/No binary judgment for bad sentences (choice order reversed) reference: '' - 5341302f-5e5c-41ac-a7cb-8843de9ad7a7: !Template + 874c6fbb-1977-45db-8791-5b1173efc30a: !Template answer_choices: A ||| B - id: 5341302f-5e5c-41ac-a7cb-8843de9ad7a7 - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] - }}' + id: 874c6fbb-1977-45db-8791-5b1173efc30a + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'A/B choice 1 (good-bad, answer: A)' + name: A/B choice randomized reference: '' - 709e0c6b-4e7f-44c7-97c4-cdcfe462ae04: !Template + 8f833560-5017-49dc-adee-53f39f9f9290: !Template answer_choices: null - id: 709e0c6b-4e7f-44c7-97c4-cdcfe462ae04 + id: 8f833560-5017-49dc-adee-53f39f9f9290 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -76,49 +108,49 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: random_choice_1_2 reference: '' - 9a386aab-430d-41b2-ab9e-af930cf8b627: !Template + abf4136a-e2c5-4630-9e3b-b9aa9f92e4e0: !Template answer_choices: A ||| B - id: 9a386aab-430d-41b2-ab9e-af930cf8b627 - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'A/B choice 2 (bad-good, answer: B)' - reference: '' - a2efd68d-dc4f-4ea6-bb2d-6717fab2dd71: !Template - answer_choices: B ||| A - id: a2efd68d-dc4f-4ea6-bb2d-6717fab2dd71 - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] - }}' + id: abf4136a-e2c5-4630-9e3b-b9aa9f92e4e0 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'B/A choice, 2 (bad-good, answer: B)' + name: A/B choice randomized (choice order reversed) reference: '' - bb2ef94c-0607-4a0e-b4b7-97d98c3b0991: !Template - answer_choices: Yes ||| No - id: bb2ef94c-0607-4a0e-b4b7-97d98c3b0991 - jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] - }} - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences - reference: '' - e8d222f9-2f64-4d17-9a9d-92f47af11868: !Template + b785ae87-1ca0-4128-a2d6-ead44ef50db9: !Template answer_choices: null - id: e8d222f9-2f64-4d17-9a9d-92f47af11868 + id: b785ae87-1ca0-4128-a2d6-ead44ef50db9 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer A or B. @@ -155,24 +187,24 @@ templates: original_task: true name: random_choice_A_B_2 reference: '' - f1e366ff-01e5-423b-8f6e-e5ec60a0981d: !Template + c49fca45-1fa3-45f7-9fe6-2ea4106fa2f0: !Template answer_choices: null - id: f1e366ff-01e5-423b-8f6e-e5ec60a0981d + id: c49fca45-1fa3-45f7-9fe6-2ea4106fa2f0 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -180,11 +212,11 @@ templates: {% if shuffled_order == 0 %} - A + 1 {% else %} - B + 2 {% endif %}' metadata: !TemplateMetadata @@ -192,26 +224,26 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: random_choice_1_2_2 reference: '' - fcd45d25-2864-4031-bb0a-941e64c4f446: !Template + c991d156-56dd-4adf-b9f7-bbc68020636b: !Template answer_choices: null - id: fcd45d25-2864-4031-bb0a-941e64c4f446 + id: c991d156-56dd-4adf-b9f7-bbc68020636b jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -219,11 +251,11 @@ templates: {% if shuffled_order == 0 %} - 1 + A {% else %} - 2 + B {% endif %}' metadata: !TemplateMetadata @@ -231,5 +263,37 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: random_choice_A_B + reference: '' + db008d32-4ee6-49be-a81c-513d7b79800e: !Template + answer_choices: Yes ||| No + id: db008d32-4ee6-49be-a81c-513d7b79800e + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences (choice order reversed) + reference: '' + e42fd39f-cb34-45c7-bb8f-37ac6d1d3a8e: !Template + answer_choices: Yes ||| No + id: e42fd39f-cb34-45c7-bb8f-37ac6d1d3a8e + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences reference: '' diff --git a/promptsource/templates/blimp/inchoative/templates.yaml b/promptsource/templates/blimp/inchoative/templates.yaml index 26634c671..db09aff5d 100644 --- a/promptsource/templates/blimp/inchoative/templates.yaml +++ b/promptsource/templates/blimp/inchoative/templates.yaml @@ -1,35 +1,133 @@ dataset: blimp subset: inchoative templates: - 5a6e8606-b48d-430d-8090-caec99767926: !Template + 17f72074-130f-4ab4-92a3-c05f3624150d: !Template + answer_choices: Yes ||| No + id: 17f72074-130f-4ab4-92a3-c05f3624150d + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences (choice order reversed) + reference: '' + 1dac225a-ac99-4cce-8385-60a79a1e58a8: !Template answer_choices: A ||| B - id: 5a6e8606-b48d-430d-8090-caec99767926 - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] - }}' + id: 1dac225a-ac99-4cce-8385-60a79a1e58a8 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'A/B choice 1 (good-bad, answer: A)' + name: A/B choice randomized + reference: '' + 1dba4dae-8844-4c1f-8978-77f97eeb9473: !Template + answer_choices: Yes ||| No + id: 1dba4dae-8844-4c1f-8978-77f97eeb9473 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences (choice order reversed) reference: '' - 5b9c18e3-2b8c-4826-b59c-736f5cf6cfdc: !Template + 52dd1d46-1d61-4af1-97fb-f576fdd74c7a: !Template answer_choices: A ||| B - id: 5b9c18e3-2b8c-4826-b59c-736f5cf6cfdc - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] - }}' + id: 52dd1d46-1d61-4af1-97fb-f576fdd74c7a + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'A/B choice 2 (bad-good, answer: B)' + name: A/B choice randomized (choice order reversed) + reference: '' + 7f5828c1-e35d-4995-a558-114eb4c44948: !Template + answer_choices: Yes ||| No + id: 7f5828c1-e35d-4995-a558-114eb4c44948 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences reference: '' - 5e219d59-5066-4947-8368-070270606712: !Template + 92e7c397-7543-4938-9df3-cff08f6bbdc3: !Template answer_choices: null - id: 5e219d59-5066-4947-8368-070270606712 + id: 92e7c397-7543-4938-9df3-cff08f6bbdc3 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer 1 or 2. @@ -66,24 +164,24 @@ templates: original_task: true name: random_choice_1_2 reference: '' - 61fbe57c-44d7-4ff2-a2e5-5946d7df4c03: !Template + 98a12cf3-be23-41dc-b746-cf75b558da5e: !Template answer_choices: null - id: 61fbe57c-44d7-4ff2-a2e5-5946d7df4c03 + id: 98a12cf3-be23-41dc-b746-cf75b558da5e jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -91,11 +189,11 @@ templates: {% if shuffled_order == 0 %} - A + 1 {% else %} - B + 2 {% endif %}' metadata: !TemplateMetadata @@ -103,23 +201,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 - reference: '' - 6fe5ff96-2d95-4d38-8ef6-9867b9b0f91f: !Template - answer_choices: Yes ||| No - id: 6fe5ff96-2d95-4d38-8ef6-9867b9b0f91f - jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] - }} - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences + name: random_choice_1_2_2 reference: '' - 8818d4d4-9a03-4fcd-ba48-434f4af6fee7: !Template + b008366e-2194-48a7-948d-d9ddcb9d67f8: !Template answer_choices: null - id: 8818d4d4-9a03-4fcd-ba48-434f4af6fee7 + id: b008366e-2194-48a7-948d-d9ddcb9d67f8 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer A or B. @@ -156,37 +242,40 @@ templates: original_task: true name: random_choice_A_B reference: '' - 9f67e427-081e-43ea-9ec2-66d48f0d7b4c: !Template - answer_choices: B ||| A - id: 9f67e427-081e-43ea-9ec2-66d48f0d7b4c - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] - }}' + f858363c-0a4b-4700-8c81-12f437359b3c: !Template + answer_choices: Yes ||| No + id: f858363c-0a4b-4700-8c81-12f437359b3c + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'B/A choice 1 (good-bad, answer: A)' + original_task: false + name: Yes/No binary judgment for good sentences reference: '' - aaabaf9c-600d-44f0-95f6-4865dc120c8c: !Template + ffcae245-ec53-4d85-a781-4323f78db347: !Template answer_choices: null - id: aaabaf9c-600d-44f0-95f6-4865dc120c8c + id: ffcae245-ec53-4d85-a781-4323f78db347 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -194,11 +283,11 @@ templates: {% if shuffled_order == 0 %} - 1 + A {% else %} - 2 + B {% endif %}' metadata: !TemplateMetadata @@ -206,30 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 - reference: '' - e0256a68-5e99-4709-bcfd-88aa6f057e11: !Template - answer_choices: Yes ||| No - id: e0256a68-5e99-4709-bcfd-88aa6f057e11 - jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] - }} - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences - reference: '' - e7d4ffcd-d5ea-4ffa-8d48-900c0156d756: !Template - answer_choices: B ||| A - id: e7d4ffcd-d5ea-4ffa-8d48-900c0156d756 - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'B/A choice, 2 (bad-good, answer: B)' + name: random_choice_A_B_2 reference: '' diff --git a/promptsource/templates/blimp/intransitive/templates.yaml b/promptsource/templates/blimp/intransitive/templates.yaml index 84cb8d38a..bf1843a0f 100644 --- a/promptsource/templates/blimp/intransitive/templates.yaml +++ b/promptsource/templates/blimp/intransitive/templates.yaml @@ -1,60 +1,102 @@ dataset: blimp subset: intransitive templates: - 18c22644-daa2-4f28-bd44-08e6a8a6baac: !Template - answer_choices: Yes ||| No - id: 18c22644-daa2-4f28-bd44-08e6a8a6baac - jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] - }} - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences - reference: '' - 445d5527-da06-41ea-9cf2-0bdbe8b0b99c: !Template + 1f1303e7-bc53-4836-b6ac-758e3eb84cdd: !Template answer_choices: A ||| B - id: 445d5527-da06-41ea-9cf2-0bdbe8b0b99c - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] - }}' + id: 1f1303e7-bc53-4836-b6ac-758e3eb84cdd + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'A/B choice 1 (good-bad, answer: A)' + name: A/B choice randomized (choice order reversed) reference: '' - 64bc634f-909c-43a7-9028-55b4950e4d28: !Template - answer_choices: B ||| A - id: 64bc634f-909c-43a7-9028-55b4950e4d28 - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] - }}' + 2b7ee9fc-ab5e-4b32-84ab-cb3418f76bf0: !Template + answer_choices: null + id: 2b7ee9fc-ab5e-4b32-84ab-cb3418f76bf0 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'B/A choice, 2 (bad-good, answer: B)' + name: random_choice_1_2 reference: '' - 757da679-3253-463c-9ad3-79a1cd8ae8bf: !Template - answer_choices: B ||| A - id: 757da679-3253-463c-9ad3-79a1cd8ae8bf - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] - }}' + 2f4d7fbd-94d2-4737-a3f1-41a92754e5ef: !Template + answer_choices: Yes ||| No + id: 2f4d7fbd-94d2-4737-a3f1-41a92754e5ef + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'B/A choice 1 (good-bad, answer: A)' + original_task: false + name: Yes/No binary judgment for good sentences (choice order reversed) reference: '' - 8472a0ce-e98b-4356-a536-bfe2be2554f7: !Template + 3c6db6ab-16bf-420c-bfba-adae7e0c4718: !Template answer_choices: null - id: 8472a0ce-e98b-4356-a536-bfe2be2554f7 + id: 3c6db6ab-16bf-420c-bfba-adae7e0c4718 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -91,9 +133,57 @@ templates: original_task: true name: random_choice_1_2_2 reference: '' - 84c781f2-77aa-4cc4-9577-5084ce3672ff: !Template + 49604859-53a6-4da5-ad45-80b5e2333885: !Template + answer_choices: Yes ||| No + id: 49604859-53a6-4da5-ad45-80b5e2333885 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' + 5487d46b-af72-4b63-9995-a8da08c151c7: !Template + answer_choices: Yes ||| No + id: 5487d46b-af72-4b63-9995-a8da08c151c7 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences (choice order reversed) + reference: '' + 83d2f463-9170-4253-9d67-bf55836360f9: !Template + answer_choices: Yes ||| No + id: 83d2f463-9170-4253-9d67-bf55836360f9 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' + aa5703bf-8cf8-4143-ae1b-b8597e6e8f3a: !Template answer_choices: null - id: 84c781f2-77aa-4cc4-9577-5084ce3672ff + id: aa5703bf-8cf8-4143-ae1b-b8597e6e8f3a jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer A or B. @@ -130,49 +220,35 @@ templates: original_task: true name: random_choice_A_B_2 reference: '' - 8b592dd2-2512-49d2-aa89-beb7ec3d0e6f: !Template + b57e5273-8571-48a3-81a4-18677792f89b: !Template answer_choices: A ||| B - id: 8b592dd2-2512-49d2-aa89-beb7ec3d0e6f - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'A/B choice 2 (bad-good, answer: B)' - reference: '' - ba9dc361-eaaf-41a7-af6d-f3930c3446fb: !Template - answer_choices: null - id: ba9dc361-eaaf-41a7-af6d-f3930c3446fb + id: b57e5273-8571-48a3-81a4-18677792f89b jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -180,26 +256,26 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: A/B choice randomized reference: '' - ec108cab-4df0-48ce-bd35-8a5e13edc4a3: !Template + b772b1a9-d804-4a3d-8767-7d8396fb3e24: !Template answer_choices: null - id: ec108cab-4df0-48ce-bd35-8a5e13edc4a3 + id: b772b1a9-d804-4a3d-8767-7d8396fb3e24 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -207,11 +283,11 @@ templates: {% if shuffled_order == 0 %} - 1 + A {% else %} - 2 + B {% endif %}' metadata: !TemplateMetadata @@ -219,17 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 - reference: '' - f5c8484d-05f8-4ea7-a629-5f746b7d2c90: !Template - answer_choices: Yes ||| No - id: f5c8484d-05f8-4ea7-a629-5f746b7d2c90 - jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] - }} - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences + name: random_choice_A_B reference: '' diff --git a/promptsource/templates/blimp/irregular_past_participle_adjectives/templates.yaml b/promptsource/templates/blimp/irregular_past_participle_adjectives/templates.yaml index 81e3f91d6..2a8f0d4e7 100644 --- a/promptsource/templates/blimp/irregular_past_participle_adjectives/templates.yaml +++ b/promptsource/templates/blimp/irregular_past_participle_adjectives/templates.yaml @@ -1,35 +1,9 @@ dataset: blimp subset: irregular_past_participle_adjectives templates: - 1a72c9d4-42eb-4967-9737-d8e98f3525b6: !Template - answer_choices: A ||| B - id: 1a72c9d4-42eb-4967-9737-d8e98f3525b6 - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'A/B choice 1 (good-bad, answer: A)' - reference: '' - 24aabf5a-7a69-4cc7-b498-10342a680928: !Template - answer_choices: B ||| A - id: 24aabf5a-7a69-4cc7-b498-10342a680928 - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'B/A choice 1 (good-bad, answer: A)' - reference: '' - 3d753b64-903d-4c1c-97ab-aec16858456f: !Template + 02dc7353-0db6-486a-aa42-d00687c9a76b: !Template answer_choices: null - id: 3d753b64-903d-4c1c-97ab-aec16858456f + id: 02dc7353-0db6-486a-aa42-d00687c9a76b jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer 1 or 2. @@ -66,25 +40,12 @@ templates: original_task: true name: random_choice_1_2 reference: '' - 9f7e3bfa-603c-4426-9f4a-2a95d6681e9d: !Template - answer_choices: A ||| B - id: 9f7e3bfa-603c-4426-9f4a-2a95d6681e9d - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'A/B choice 2 (bad-good, answer: B)' - reference: '' - cacb755f-525a-4d9c-bfd2-0ff72bdfde36: !Template + 0806fff0-7705-4703-a2a3-09c1c5eccd88: !Template answer_choices: null - id: cacb755f-525a-4d9c-bfd2-0ff72bdfde36 + id: 0806fff0-7705-4703-a2a3-09c1c5eccd88 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -116,26 +77,53 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: random_choice_A_B reference: '' - d64cf1fe-9435-40fc-b1b9-a7375f75940d: !Template - answer_choices: Yes ||| No - id: d64cf1fe-9435-40fc-b1b9-a7375f75940d - jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] - }} + 0850c29e-05ab-4a0e-b05a-46d77a610ebf: !Template + answer_choices: null + id: 0850c29e-05ab-4a0e-b05a-46d77a610ebf + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' metadata: !TemplateMetadata - choices_in_prompt: false + choices_in_prompt: true metrics: - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences + original_task: true + name: random_choice_1_2_2 reference: '' - df8909c4-d785-436e-8b15-d60bccdd8999: !Template + 1b6f1d5e-6285-4e26-9cf3-88c6c4e00874: !Template answer_choices: null - id: df8909c4-d785-436e-8b15-d60bccdd8999 + id: 1b6f1d5e-6285-4e26-9cf3-88c6c4e00874 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -167,63 +155,91 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: random_choice_A_B_2 reference: '' - e4101a58-a7f8-4df5-8f88-c92b2efd47c2: !Template - answer_choices: Yes ||| No - id: e4101a58-a7f8-4df5-8f88-c92b2efd47c2 - jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] - }} + 23cbd23b-8e98-4c4f-8730-022a6dd1cadb: !Template + answer_choices: A ||| B + id: 23cbd23b-8e98-4c4f-8730-022a6dd1cadb + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata - choices_in_prompt: false + choices_in_prompt: true metrics: - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences + original_task: true + name: A/B choice randomized (choice order reversed) reference: '' - ee6ebb5a-49a2-4e9e-b34e-8f95607dc9d1: !Template - answer_choices: B ||| A - id: ee6ebb5a-49a2-4e9e-b34e-8f95607dc9d1 - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] - }}' + 278c50d2-ea29-4ea7-95c8-0ff8a95e943a: !Template + answer_choices: Yes ||| No + id: 278c50d2-ea29-4ea7-95c8-0ff8a95e943a + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'B/A choice, 2 (bad-good, answer: B)' + original_task: false + name: Yes/No binary judgment for bad sentences reference: '' - fbb1facd-f57b-47b7-82b3-e107bbc20e65: !Template - answer_choices: null - id: fbb1facd-f57b-47b7-82b3-e107bbc20e65 + 2e9e9051-6f10-4862-8328-2db12b1d85e8: !Template + answer_choices: A ||| B + id: 2e9e9051-6f10-4862-8328-2db12b1d85e8 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} - {% if shuffled_order == 0 %} - - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -231,5 +247,53 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: A/B choice randomized + reference: '' + 3a8e9ab1-0304-4813-82b5-b3635ada8862: !Template + answer_choices: Yes ||| No + id: 3a8e9ab1-0304-4813-82b5-b3635ada8862 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' + 4f93965c-9453-4d4c-8a31-e229a0c49a23: !Template + answer_choices: Yes ||| No + id: 4f93965c-9453-4d4c-8a31-e229a0c49a23 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences (choice order reversed) + reference: '' + 9bc7df13-b5a9-453b-a286-c6fb8509b58f: !Template + answer_choices: Yes ||| No + id: 9bc7df13-b5a9-453b-a286-c6fb8509b58f + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences (choice order reversed) reference: '' diff --git a/promptsource/templates/blimp/irregular_past_participle_verbs/templates.yaml b/promptsource/templates/blimp/irregular_past_participle_verbs/templates.yaml index bdf0265a1..3ea908ea9 100644 --- a/promptsource/templates/blimp/irregular_past_participle_verbs/templates.yaml +++ b/promptsource/templates/blimp/irregular_past_participle_verbs/templates.yaml @@ -1,46 +1,25 @@ dataset: blimp subset: irregular_past_participle_verbs templates: - 28aa62d4-83b4-42d8-b090-0557c0d7132a: !Template + 1d8c20e1-b160-4500-b0b2-99d91f09d718: !Template answer_choices: Yes ||| No - id: 28aa62d4-83b4-42d8-b090-0557c0d7132a - jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] - }} - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences - reference: '' - 3c661493-e0ce-4469-a1c6-672172ad4362: !Template - answer_choices: B ||| A - id: 3c661493-e0ce-4469-a1c6-672172ad4362 - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] - }}' + id: 1d8c20e1-b160-4500-b0b2-99d91f09d718 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'B/A choice, 2 (bad-good, answer: B)' - reference: '' - 4904ebe5-2da9-40b7-bfb7-10a6775a8eb0: !Template - answer_choices: Yes ||| No - id: 4904ebe5-2da9-40b7-bfb7-10a6775a8eb0 - jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] - }} - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy original_task: false - name: Yes/No binary judgment for bad sentences + name: Yes/No binary judgment for good sentences reference: '' - 51c3e1c5-02c8-4ede-9766-d8dd8b24ce68: !Template + 2207891a-b466-449c-8a79-15c5b97ec5ae: !Template answer_choices: null - id: 51c3e1c5-02c8-4ede-9766-d8dd8b24ce68 + id: 2207891a-b466-449c-8a79-15c5b97ec5ae jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer A or B. @@ -77,25 +56,12 @@ templates: original_task: true name: random_choice_A_B_2 reference: '' - 7c96cdf3-38a3-40ce-a3d1-c747af08902c: !Template - answer_choices: B ||| A - id: 7c96cdf3-38a3-40ce-a3d1-c747af08902c - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'B/A choice 1 (good-bad, answer: A)' - reference: '' - 8d7fd6ac-98bc-46b4-99ea-2582384a89e7: !Template + 375f0be0-e7ac-4c80-a9db-e279c42f4f27: !Template answer_choices: null - id: 8d7fd6ac-98bc-46b4-99ea-2582384a89e7 + id: 375f0be0-e7ac-4c80-a9db-e279c42f4f27 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -127,11 +93,65 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: random_choice_1_2_2 + reference: '' + 40b0b799-f6a6-4f67-aaf2-f9ddebe3fe32: !Template + answer_choices: Yes ||| No + id: 40b0b799-f6a6-4f67-aaf2-f9ddebe3fe32 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences (choice order reversed) + reference: '' + 4896f5d2-f044-4a4e-a222-329c2740d2af: !Template + answer_choices: A ||| B + id: 4896f5d2-f044-4a4e-a222-329c2740d2af + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: A/B choice randomized (choice order reversed) reference: '' - 93989e52-d946-4bb2-b0c3-63cbf8971333: !Template + 6351922c-0b3f-4e0c-bee9-84052a48861e: !Template answer_choices: null - id: 93989e52-d946-4bb2-b0c3-63cbf8971333 + id: 6351922c-0b3f-4e0c-bee9-84052a48861e jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer A or B. @@ -168,62 +188,106 @@ templates: original_task: true name: random_choice_A_B reference: '' - b57f9e72-5ed7-4702-810b-bbe5e0fe17d6: !Template - answer_choices: A ||| B - id: b57f9e72-5ed7-4702-810b-bbe5e0fe17d6 - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] - }}' + 74a843fc-2f3a-408d-8717-fba1fd4a9a81: !Template + answer_choices: null + id: 74a843fc-2f3a-408d-8717-fba1fd4a9a81 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'A/B choice 1 (good-bad, answer: A)' + name: random_choice_1_2 reference: '' - b74c602d-5a1f-49da-9441-5bce4187e0b6: !Template - answer_choices: A ||| B - id: b74c602d-5a1f-49da-9441-5bce4187e0b6 - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] - }}' + c2b54abf-0de1-4be0-ade2-998e3b7d3d2e: !Template + answer_choices: Yes ||| No + id: c2b54abf-0de1-4be0-ade2-998e3b7d3d2e + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'A/B choice 2 (bad-good, answer: B)' + original_task: false + name: Yes/No binary judgment for bad sentences reference: '' - d016c1e6-3545-4825-bae9-9cc25b6e14f1: !Template - answer_choices: null - id: d016c1e6-3545-4825-bae9-9cc25b6e14f1 + c909bbc2-fc9c-4d05-9367-cce7ad261b4f: !Template + answer_choices: Yes ||| No + id: c909bbc2-fc9c-4d05-9367-cce7ad261b4f + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences (choice order reversed) + reference: '' + f5f2ac7f-4daa-48f2-b376-2092bd7044f5: !Template + answer_choices: A ||| B + id: f5f2ac7f-4daa-48f2-b376-2092bd7044f5 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} - {% if shuffled_order == 0 %} - - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -231,5 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: A/B choice randomized reference: '' diff --git a/promptsource/templates/blimp/irregular_plural_subject_verb_agreement_1/templates.yaml b/promptsource/templates/blimp/irregular_plural_subject_verb_agreement_1/templates.yaml index 47ff76103..4908ee1f7 100644 --- a/promptsource/templates/blimp/irregular_plural_subject_verb_agreement_1/templates.yaml +++ b/promptsource/templates/blimp/irregular_plural_subject_verb_agreement_1/templates.yaml @@ -1,47 +1,80 @@ dataset: blimp subset: irregular_plural_subject_verb_agreement_1 templates: - 06e6d299-ddd2-45e6-8ec3-3c0bf9774323: !Template - answer_choices: A ||| B - id: 06e6d299-ddd2-45e6-8ec3-3c0bf9774323 - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] - }}' + 07969670-5712-41fc-ae94-23a48da04c23: !Template + answer_choices: Yes ||| No + id: 07969670-5712-41fc-ae94-23a48da04c23 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'A/B choice 1 (good-bad, answer: A)' + original_task: false + name: Yes/No binary judgment for bad sentences (choice order reversed) reference: '' - 212670be-9b84-4386-beec-bf09dbb31161: !Template + 138928eb-3b09-42f8-9e6b-f45b6533919d: !Template answer_choices: Yes ||| No - id: 212670be-9b84-4386-beec-bf09dbb31161 - jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] - }} + id: 138928eb-3b09-42f8-9e6b-f45b6533919d + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata - choices_in_prompt: false + choices_in_prompt: true metrics: - Accuracy original_task: false - name: Yes/No binary judgment for good sentences + name: Yes/No binary judgment for bad sentences reference: '' - 3de6f05b-6c60-48c1-b91a-b10b9dadb022: !Template - answer_choices: A ||| B - id: 3de6f05b-6c60-48c1-b91a-b10b9dadb022 - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] - }}' + 15d22e3c-1265-4ce4-98df-9236e34bca27: !Template + answer_choices: null + id: 15d22e3c-1265-4ce4-98df-9236e34bca27 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'A/B choice 2 (bad-good, answer: B)' + name: random_choice_1_2_2 reference: '' - 737cba97-50ce-4f61-9882-fd98e361a3dd: !Template + 32abd943-3331-45dc-ba1f-00b8603cac53: !Template answer_choices: null - id: 737cba97-50ce-4f61-9882-fd98e361a3dd + id: 32abd943-3331-45dc-ba1f-00b8603cac53 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer 1 or 2. @@ -78,25 +111,50 @@ templates: original_task: true name: random_choice_1_2 reference: '' - 784efb6b-2959-48df-b9a7-8ccad19b1f11: !Template - answer_choices: B ||| A - id: 784efb6b-2959-48df-b9a7-8ccad19b1f11 - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] - }}' + 5e1bb1ec-ee7b-4535-83e6-e5bb23f4ca44: !Template + answer_choices: A ||| B + id: 5e1bb1ec-ee7b-4535-83e6-e5bb23f4ca44 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'B/A choice 1 (good-bad, answer: A)' + name: A/B choice randomized (choice order reversed) reference: '' - 9a7885e6-9381-4d96-8614-3e3307627e85: !Template + 732e9a51-add7-4726-aea6-a133527cec8b: !Template answer_choices: null - id: 9a7885e6-9381-4d96-8614-3e3307627e85 + id: 732e9a51-add7-4726-aea6-a133527cec8b jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -128,51 +186,37 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B - reference: '' - b8ece796-4f9c-4176-b809-319c85ba8083: !Template - answer_choices: B ||| A - id: b8ece796-4f9c-4176-b809-319c85ba8083 - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'B/A choice, 2 (bad-good, answer: B)' + name: random_choice_A_B_2 reference: '' - d20bcffc-1bd0-4058-8101-c35cdafde31a: !Template - answer_choices: null - id: d20bcffc-1bd0-4058-8101-c35cdafde31a + bc17aab3-1102-4e38-899f-b8f8a10c2699: !Template + answer_choices: A ||| B + id: bc17aab3-1102-4e38-899f-b8f8a10c2699 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} - {% if shuffled_order == 0 %} - - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -180,26 +224,30 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: A/B choice randomized reference: '' - e2c2996c-d9c9-4085-b945-953ba5677520: !Template + cb17bb57-f035-4a9b-afc0-e010634d4c51: !Template answer_choices: Yes ||| No - id: e2c2996c-d9c9-4085-b945-953ba5677520 - jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] - }} + id: cb17bb57-f035-4a9b-afc0-e010634d4c51 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata - choices_in_prompt: false + choices_in_prompt: true metrics: - Accuracy original_task: false - name: Yes/No binary judgment for bad sentences + name: Yes/No binary judgment for good sentences reference: '' - f6733076-3461-4522-b5cd-38cc50469c48: !Template + cc7203ea-8c85-40a8-a778-baa94467b25c: !Template answer_choices: null - id: f6733076-3461-4522-b5cd-38cc50469c48 + id: cc7203ea-8c85-40a8-a778-baa94467b25c jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -231,5 +279,21 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: random_choice_A_B + reference: '' + d0884cf3-1420-4064-981d-7daebe5b933a: !Template + answer_choices: Yes ||| No + id: d0884cf3-1420-4064-981d-7daebe5b933a + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences (choice order reversed) reference: '' diff --git a/promptsource/templates/blimp/irregular_plural_subject_verb_agreement_2/templates.yaml b/promptsource/templates/blimp/irregular_plural_subject_verb_agreement_2/templates.yaml index df3e571d9..09acd2d89 100644 --- a/promptsource/templates/blimp/irregular_plural_subject_verb_agreement_2/templates.yaml +++ b/promptsource/templates/blimp/irregular_plural_subject_verb_agreement_2/templates.yaml @@ -1,47 +1,63 @@ dataset: blimp subset: irregular_plural_subject_verb_agreement_2 templates: - 0cb636cc-79c8-41d5-830c-c5b6e911152c: !Template + 29407110-06ab-4aab-8c8b-2d025da8bcad: !Template answer_choices: A ||| B - id: 0cb636cc-79c8-41d5-830c-c5b6e911152c - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] - }}' + id: 29407110-06ab-4aab-8c8b-2d025da8bcad + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'A/B choice 1 (good-bad, answer: A)' + name: A/B choice randomized (choice order reversed) reference: '' - 2309b89d-aa5b-42e1-a3a0-18df116d1b89: !Template + 2ef81a47-e0e5-43ac-84a6-4587760a3da9: !Template answer_choices: Yes ||| No - id: 2309b89d-aa5b-42e1-a3a0-18df116d1b89 - jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] - }} - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences - reference: '' - 5a5953f1-1122-4c42-9013-a86507bd70c6: !Template - answer_choices: A ||| B - id: 5a5953f1-1122-4c42-9013-a86507bd70c6 - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] - }}' + id: 2ef81a47-e0e5-43ac-84a6-4587760a3da9 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'A/B choice 2 (bad-good, answer: B)' + original_task: false + name: Yes/No binary judgment for bad sentences (choice order reversed) reference: '' - 602abc32-08ac-4f01-a290-7c334c5b0275: !Template + 46512145-cd20-4d93-bba7-d3e403683925: !Template answer_choices: null - id: 602abc32-08ac-4f01-a290-7c334c5b0275 + id: 46512145-cd20-4d93-bba7-d3e403683925 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer A or B. @@ -78,21 +94,48 @@ templates: original_task: true name: random_choice_A_B_2 reference: '' - 6a067177-3527-47b1-9667-852d967591c0: !Template - answer_choices: Yes ||| No - id: 6a067177-3527-47b1-9667-852d967591c0 - jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] - }} + 5e1a0869-264a-4870-804b-542499923f9f: !Template + answer_choices: null + id: 5e1a0869-264a-4870-804b-542499923f9f + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' metadata: !TemplateMetadata - choices_in_prompt: false + choices_in_prompt: true metrics: - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences + original_task: true + name: random_choice_1_2 reference: '' - 923eb5b8-43e3-472c-b5bb-70f3422bec0a: !Template + 65b77969-a0cf-472c-9c0b-437acae763a7: !Template answer_choices: null - id: 923eb5b8-43e3-472c-b5bb-70f3422bec0a + id: 65b77969-a0cf-472c-9c0b-437acae763a7 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -129,9 +172,9 @@ templates: original_task: true name: random_choice_1_2_2 reference: '' - 99dd4962-0a8b-4f2e-82dd-41180a15e3ca: !Template + 7ed1f1fe-25e7-43eb-abd8-9dcc1146d0f2: !Template answer_choices: null - id: 99dd4962-0a8b-4f2e-82dd-41180a15e3ca + id: 7ed1f1fe-25e7-43eb-abd8-9dcc1146d0f2 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer A or B. @@ -168,62 +211,35 @@ templates: original_task: true name: random_choice_A_B reference: '' - b06e97b2-0d79-4e16-96da-444683e45b71: !Template - answer_choices: B ||| A - id: b06e97b2-0d79-4e16-96da-444683e45b71 - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'B/A choice, 2 (bad-good, answer: B)' - reference: '' - b4a15970-4849-4b6b-b3c0-a93a8181b385: !Template - answer_choices: B ||| A - id: b4a15970-4849-4b6b-b3c0-a93a8181b385 - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'B/A choice 1 (good-bad, answer: A)' - reference: '' - d9e70db4-80d1-47af-a029-92082aa0739f: !Template - answer_choices: null - id: d9e70db4-80d1-47af-a029-92082aa0739f + 88d5eb31-2be6-44ff-82be-5263900c5159: !Template + answer_choices: A ||| B + id: 88d5eb31-2be6-44ff-82be-5263900c5159 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -231,5 +247,53 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: A/B choice randomized + reference: '' + 94ca08bd-aa2d-4317-8366-dd85932d455a: !Template + answer_choices: Yes ||| No + id: 94ca08bd-aa2d-4317-8366-dd85932d455a + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' + a6701d62-53f9-4fad-ac3c-745a8842fa59: !Template + answer_choices: Yes ||| No + id: a6701d62-53f9-4fad-ac3c-745a8842fa59 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences (choice order reversed) + reference: '' + fa23d667-6719-4b80-94c1-0cd563c8fc2d: !Template + answer_choices: Yes ||| No + id: fa23d667-6719-4b80-94c1-0cd563c8fc2d + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences reference: '' diff --git a/promptsource/templates/blimp/left_branch_island_echo_question/templates.yaml b/promptsource/templates/blimp/left_branch_island_echo_question/templates.yaml index c518ed84e..723796ce3 100644 --- a/promptsource/templates/blimp/left_branch_island_echo_question/templates.yaml +++ b/promptsource/templates/blimp/left_branch_island_echo_question/templates.yaml @@ -1,9 +1,9 @@ dataset: blimp subset: left_branch_island_echo_question templates: - 2a0d77a0-689c-4871-aabb-658c8be08dbb: !Template + 0eceb998-0f5d-4bb5-b540-3233d7f698cb: !Template answer_choices: null - id: 2a0d77a0-689c-4871-aabb-658c8be08dbb + id: 0eceb998-0f5d-4bb5-b540-3233d7f698cb jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer A or B. @@ -40,37 +40,60 @@ templates: original_task: true name: random_choice_A_B_2 reference: '' - 59cce349-5011-46f2-9403-e641d5eea89e: !Template - answer_choices: B ||| A - id: 59cce349-5011-46f2-9403-e641d5eea89e - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] - }}' + 1c8d3485-79db-4d64-986b-87b885b3daa2: !Template + answer_choices: Yes ||| No + id: 1c8d3485-79db-4d64-986b-87b885b3daa2 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'B/A choice, 2 (bad-good, answer: B)' + original_task: false + name: Yes/No binary judgment for good sentences (choice order reversed) reference: '' - 5c0750d9-45fa-45f0-92ab-82916134f126: !Template + 1d543084-035c-45a5-a09c-21dd4678059c: !Template answer_choices: Yes ||| No - id: 5c0750d9-45fa-45f0-92ab-82916134f126 - jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] - }} + id: 1d543084-035c-45a5-a09c-21dd4678059c + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata - choices_in_prompt: false + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' + 4383c751-d28e-4bb3-a7cc-eed29a3a984d: !Template + answer_choices: Yes ||| No + id: 4383c751-d28e-4bb3-a7cc-eed29a3a984d + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true metrics: - Accuracy original_task: false name: Yes/No binary judgment for good sentences reference: '' - 70131810-2398-496a-903a-b518f2193f38: !Template + 778bad58-dcc3-4dd6-b384-465094df51bd: !Template answer_choices: null - id: 70131810-2398-496a-903a-b518f2193f38 + id: 778bad58-dcc3-4dd6-b384-465094df51bd jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -102,11 +125,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: random_choice_1_2_2 reference: '' - 7893c399-c3dd-4515-baa1-59dd8c28c265: !Template + 921d7113-a724-4849-8f92-af8cf83e062c: !Template answer_choices: null - id: 7893c399-c3dd-4515-baa1-59dd8c28c265 + id: 921d7113-a724-4849-8f92-af8cf83e062c jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer A or B. @@ -143,74 +166,90 @@ templates: original_task: true name: random_choice_A_B reference: '' - 8a09370c-d922-416c-b0b5-0cb8cfeddef5: !Template - answer_choices: B ||| A - id: 8a09370c-d922-416c-b0b5-0cb8cfeddef5 - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'B/A choice 1 (good-bad, answer: A)' - reference: '' - 918d5796-3bd1-4492-96d4-c63c6ee418b0: !Template - answer_choices: A ||| B - id: 918d5796-3bd1-4492-96d4-c63c6ee418b0 - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] - }}' + a8e82ebd-d361-4129-9bba-04ab08545c9b: !Template + answer_choices: null + id: a8e82ebd-d361-4129-9bba-04ab08545c9b + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'A/B choice 1 (good-bad, answer: A)' + name: random_choice_1_2 reference: '' - acee5cf1-dd6c-41e0-9b7d-d756893ceb21: !Template + b58d6246-9221-43fe-bea5-4ff966cb8563: !Template answer_choices: Yes ||| No - id: acee5cf1-dd6c-41e0-9b7d-d756893ceb21 - jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] - }} + id: b58d6246-9221-43fe-bea5-4ff966cb8563 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata - choices_in_prompt: false + choices_in_prompt: true metrics: - Accuracy original_task: false - name: Yes/No binary judgment for bad sentences + name: Yes/No binary judgment for bad sentences (choice order reversed) reference: '' - d69689cf-0926-4ac8-9b77-8e8d88e88747: !Template - answer_choices: null - id: d69689cf-0926-4ac8-9b77-8e8d88e88747 + ba91c932-61de-4b29-b2d7-551f3f0dfa8f: !Template + answer_choices: A ||| B + id: ba91c932-61de-4b29-b2d7-551f3f0dfa8f jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -218,18 +257,43 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: A/B choice randomized reference: '' - ff26de8b-4911-47f3-bc96-777c14320e82: !Template + d73bced1-ba60-48ae-b958-a8fb818bfc68: !Template answer_choices: A ||| B - id: ff26de8b-4911-47f3-bc96-777c14320e82 - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] - }}' + id: d73bced1-ba60-48ae-b958-a8fb818bfc68 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'A/B choice 2 (bad-good, answer: B)' + name: A/B choice randomized (choice order reversed) reference: '' diff --git a/promptsource/templates/blimp/left_branch_island_simple_question/templates.yaml b/promptsource/templates/blimp/left_branch_island_simple_question/templates.yaml index 9d820fd38..fa8c761f4 100644 --- a/promptsource/templates/blimp/left_branch_island_simple_question/templates.yaml +++ b/promptsource/templates/blimp/left_branch_island_simple_question/templates.yaml @@ -1,9 +1,9 @@ dataset: blimp subset: left_branch_island_simple_question templates: - 0b748807-1d95-4764-8b42-8b45bb26ddeb: !Template + 3993c5b2-dd53-4412-892b-78c84241d2dc: !Template answer_choices: null - id: 0b748807-1d95-4764-8b42-8b45bb26ddeb + id: 3993c5b2-dd53-4412-892b-78c84241d2dc jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer A or B. @@ -40,24 +40,62 @@ templates: original_task: true name: random_choice_A_B reference: '' - 11975bac-b906-426d-ac75-1f00d6556a8f: !Template + 41b1a01f-2115-4303-bb4d-95431bafe7f1: !Template + answer_choices: A ||| B + id: 41b1a01f-2115-4303-bb4d-95431bafe7f1 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: A/B choice randomized + reference: '' + 66eb418d-7e90-4d03-bff7-58df497417ee: !Template answer_choices: null - id: 11975bac-b906-426d-ac75-1f00d6556a8f + id: 66eb418d-7e90-4d03-bff7-58df497417ee jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -65,11 +103,11 @@ templates: {% if shuffled_order == 0 %} - 1 + A {% else %} - 2 + B {% endif %}' metadata: !TemplateMetadata @@ -77,37 +115,81 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: random_choice_A_B_2 reference: '' - 28ee916d-b379-4757-a016-74a18e36cf01: !Template - answer_choices: A ||| B - id: 28ee916d-b379-4757-a016-74a18e36cf01 - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] - }}' + 6e3c956d-f4aa-45aa-9f37-c0d61dac5976: !Template + answer_choices: Yes ||| No + id: 6e3c956d-f4aa-45aa-9f37-c0d61dac5976 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'A/B choice 1 (good-bad, answer: A)' + original_task: false + name: Yes/No binary judgment for good sentences (choice order reversed) + reference: '' + 6fa62282-0461-458a-9ade-1b146097f5ee: !Template + answer_choices: Yes ||| No + id: 6fa62282-0461-458a-9ade-1b146097f5ee + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences (choice order reversed) reference: '' - 3cdd553b-aae7-4a6a-9086-13b11ee1793e: !Template - answer_choices: B ||| A - id: 3cdd553b-aae7-4a6a-9086-13b11ee1793e - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] - }}' + 8eb6c647-eb5e-46f5-9bd8-f17d792be791: !Template + answer_choices: A ||| B + id: 8eb6c647-eb5e-46f5-9bd8-f17d792be791 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'B/A choice 1 (good-bad, answer: A)' + name: A/B choice randomized (choice order reversed) reference: '' - 48640caf-c52c-4c14-985e-592ca3ddab7d: !Template + 935b11ae-afd6-44bf-8fa8-42f0650795d4: !Template answer_choices: null - id: 48640caf-c52c-4c14-985e-592ca3ddab7d + id: 935b11ae-afd6-44bf-8fa8-42f0650795d4 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -144,49 +226,40 @@ templates: original_task: true name: random_choice_1_2_2 reference: '' - 4c2541de-62eb-4e42-a16a-043ea5b4a592: !Template + cb3a865e-4f7c-4595-be43-c6586608bdb8: !Template answer_choices: Yes ||| No - id: 4c2541de-62eb-4e42-a16a-043ea5b4a592 - jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] - }} - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences - reference: '' - 62267fed-2552-489f-9426-659c797d89c2: !Template - answer_choices: A ||| B - id: 62267fed-2552-489f-9426-659c797d89c2 - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] - }}' + id: cb3a865e-4f7c-4595-be43-c6586608bdb8 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'A/B choice 2 (bad-good, answer: B)' + original_task: false + name: Yes/No binary judgment for good sentences reference: '' - 932e58d7-cee2-490e-bc82-a94c9e1e81c1: !Template + e9b6032e-6723-454c-be08-15f45d3471dd: !Template answer_choices: null - id: 932e58d7-cee2-490e-bc82-a94c9e1e81c1 + id: e9b6032e-6723-454c-be08-15f45d3471dd jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -194,11 +267,11 @@ templates: {% if shuffled_order == 0 %} - A + 1 {% else %} - B + 2 {% endif %}' metadata: !TemplateMetadata @@ -206,30 +279,21 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: random_choice_1_2 reference: '' - b621a773-8012-40ac-8302-98e930395200: !Template + fa93c786-5e75-4039-86f6-d7a67423a2bd: !Template answer_choices: Yes ||| No - id: b621a773-8012-40ac-8302-98e930395200 - jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] - }} - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences - reference: '' - e6c7fbb0-65ed-4faa-9d47-0f980535c75d: !Template - answer_choices: B ||| A - id: e6c7fbb0-65ed-4faa-9d47-0f980535c75d - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] - }}' + id: fa93c786-5e75-4039-86f6-d7a67423a2bd + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'B/A choice, 2 (bad-good, answer: B)' + original_task: false + name: Yes/No binary judgment for bad sentences reference: '' diff --git a/promptsource/templates/blimp/matrix_question_npi_licensor_present/templates.yaml b/promptsource/templates/blimp/matrix_question_npi_licensor_present/templates.yaml index f81c128cc..cf91bf26c 100644 --- a/promptsource/templates/blimp/matrix_question_npi_licensor_present/templates.yaml +++ b/promptsource/templates/blimp/matrix_question_npi_licensor_present/templates.yaml @@ -1,72 +1,25 @@ dataset: blimp subset: matrix_question_npi_licensor_present templates: - 2ee4a89b-4a80-4639-8a6f-7798141ac45b: !Template - answer_choices: B ||| A - id: 2ee4a89b-4a80-4639-8a6f-7798141ac45b - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'B/A choice, 2 (bad-good, answer: B)' - reference: '' - 353a20b7-daf7-43c4-8a9d-356a039b8422: !Template + 0b1142ff-05ec-4e72-98b5-690a28ef370d: !Template answer_choices: Yes ||| No - id: 353a20b7-daf7-43c4-8a9d-356a039b8422 - jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] - }} - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences - reference: '' - 38467926-4540-4d7d-853d-fd7c4cd31519: !Template - answer_choices: A ||| B - id: 38467926-4540-4d7d-853d-fd7c4cd31519 - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'A/B choice 2 (bad-good, answer: B)' - reference: '' - 41c29d52-e4e5-4ec7-86f9-d94e06d72a3d: !Template - answer_choices: A ||| B - id: 41c29d52-e4e5-4ec7-86f9-d94e06d72a3d - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] - }}' + id: 0b1142ff-05ec-4e72-98b5-690a28ef370d + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'A/B choice 1 (good-bad, answer: A)' - reference: '' - 514cb156-b57e-43b0-bc58-ae6439487c7d: !Template - answer_choices: Yes ||| No - id: 514cb156-b57e-43b0-bc58-ae6439487c7d - jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] - }} - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy original_task: false - name: Yes/No binary judgment for bad sentences + name: Yes/No binary judgment for bad sentences (choice order reversed) reference: '' - 75c48a8a-96ab-4fe6-b84f-a018e29a6e6a: !Template + 0f3a0902-5ddd-4097-bf93-db6c8e1d50ec: !Template answer_choices: null - id: 75c48a8a-96ab-4fe6-b84f-a018e29a6e6a + id: 0f3a0902-5ddd-4097-bf93-db6c8e1d50ec jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer 1 or 2. @@ -103,12 +56,28 @@ templates: original_task: true name: random_choice_1_2 reference: '' - 80972308-0476-4ea4-9262-8d3ea7fcbfac: !Template + 3fb53f8e-11cb-42ca-8c78-8587dc312f0a: !Template + answer_choices: Yes ||| No + id: 3fb53f8e-11cb-42ca-8c78-8587dc312f0a + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' + 5dfb1a1d-b765-4b98-b86f-738cd92d9d25: !Template answer_choices: null - id: 80972308-0476-4ea4-9262-8d3ea7fcbfac + id: 5dfb1a1d-b765-4b98-b86f-738cd92d9d25 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -140,11 +109,27 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: random_choice_A_B reference: '' - b41aaa31-e18d-4168-a419-1ad39b503ba7: !Template + 62edcb10-abb0-47df-8102-10cc3be331f7: !Template + answer_choices: Yes ||| No + id: 62edcb10-abb0-47df-8102-10cc3be331f7 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences (choice order reversed) + reference: '' + 65f75f9e-3636-4fd6-a406-b9d0d44897c2: !Template answer_choices: null - id: b41aaa31-e18d-4168-a419-1ad39b503ba7 + id: 65f75f9e-3636-4fd6-a406-b9d0d44897c2 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -181,25 +166,66 @@ templates: original_task: true name: random_choice_1_2_2 reference: '' - cfb7e5a0-9a33-49c0-bfeb-9db76af3174a: !Template - answer_choices: B ||| A - id: cfb7e5a0-9a33-49c0-bfeb-9db76af3174a - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] - }}' + 91f671f7-52e2-432d-9bcc-18db6a5ed691: !Template + answer_choices: A ||| B + id: 91f671f7-52e2-432d-9bcc-18db6a5ed691 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'B/A choice 1 (good-bad, answer: A)' + name: A/B choice randomized (choice order reversed) reference: '' - f63c9f36-330c-4153-9dc4-79ce590de043: !Template + a401118b-af0e-4f52-ac3a-b057acd65d1e: !Template + answer_choices: Yes ||| No + id: a401118b-af0e-4f52-ac3a-b057acd65d1e + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' + a91a1346-df74-4f7d-b188-6fbcd472a9b0: !Template answer_choices: null - id: f63c9f36-330c-4153-9dc4-79ce590de043 + id: a91a1346-df74-4f7d-b188-6fbcd472a9b0 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -231,5 +257,43 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: random_choice_A_B_2 + reference: '' + cf4b9b83-cc0f-4c2e-b923-06f099b56c37: !Template + answer_choices: A ||| B + id: cf4b9b83-cc0f-4c2e-b923-06f099b56c37 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: A/B choice randomized reference: '' diff --git a/promptsource/templates/blimp/npi_present_1/templates.yaml b/promptsource/templates/blimp/npi_present_1/templates.yaml index 8a1d32bb4..e0053b4ca 100644 --- a/promptsource/templates/blimp/npi_present_1/templates.yaml +++ b/promptsource/templates/blimp/npi_present_1/templates.yaml @@ -1,85 +1,134 @@ dataset: blimp subset: npi_present_1 templates: - 168e8577-4d03-483c-8c6d-a03163582af3: !Template - answer_choices: Yes ||| No - id: 168e8577-4d03-483c-8c6d-a03163582af3 - jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] - }} + 24f2d528-aa35-4f44-bb71-7f8f30716ff5: !Template + answer_choices: null + id: 24f2d528-aa35-4f44-bb71-7f8f30716ff5 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' metadata: !TemplateMetadata - choices_in_prompt: false + choices_in_prompt: true metrics: - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences + original_task: true + name: random_choice_A_B reference: '' - 3bc65cb7-9340-4d86-b8e0-523bb7db0291: !Template - answer_choices: B ||| A - id: 3bc65cb7-9340-4d86-b8e0-523bb7db0291 - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] - }}' + 31bc6a1e-cbed-4b52-9f81-774b035251d4: !Template + answer_choices: Yes ||| No + id: 31bc6a1e-cbed-4b52-9f81-774b035251d4 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'B/A choice, 2 (bad-good, answer: B)' + original_task: false + name: Yes/No binary judgment for bad sentences reference: '' - 514edf8b-5e8d-48d5-9176-178271e6be8e: !Template - answer_choices: A ||| B - id: 514edf8b-5e8d-48d5-9176-178271e6be8e - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] - }}' + 3c54a3db-f8ea-496c-90d9-80493742adcb: !Template + answer_choices: Yes ||| No + id: 3c54a3db-f8ea-496c-90d9-80493742adcb + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'A/B choice 1 (good-bad, answer: A)' + original_task: false + name: Yes/No binary judgment for good sentences (choice order reversed) reference: '' - 545cd705-4763-45bf-8cc3-5a5e1d51f6fe: !Template - answer_choices: B ||| A - id: 545cd705-4763-45bf-8cc3-5a5e1d51f6fe - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] - }}' + 70b6b55d-e052-4664-a60e-8da216aa9b10: !Template + answer_choices: A ||| B + id: 70b6b55d-e052-4664-a60e-8da216aa9b10 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'B/A choice 1 (good-bad, answer: A)' + name: A/B choice randomized (choice order reversed) reference: '' - 58c85c61-1fc3-4b00-9e8b-4f6a24b0f9c8: !Template + 76542a82-0e43-47f0-bf91-5908e3baab16: !Template answer_choices: Yes ||| No - id: 58c85c61-1fc3-4b00-9e8b-4f6a24b0f9c8 - jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] - }} - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences - reference: '' - 5b2a209b-90c6-4d0e-b08f-ddf5e2755b8b: !Template - answer_choices: A ||| B - id: 5b2a209b-90c6-4d0e-b08f-ddf5e2755b8b - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] - }}' + id: 76542a82-0e43-47f0-bf91-5908e3baab16 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'A/B choice 2 (bad-good, answer: B)' + original_task: false + name: Yes/No binary judgment for bad sentences (choice order reversed) reference: '' - 6cb52fe9-6c71-414f-ac75-b30e1f60a215: !Template + 7dc5783a-0d77-4c1b-bb3b-9737c45f1860: !Template answer_choices: null - id: 6cb52fe9-6c71-414f-ac75-b30e1f60a215 + id: 7dc5783a-0d77-4c1b-bb3b-9737c45f1860 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer 1 or 2. @@ -116,36 +165,35 @@ templates: original_task: true name: random_choice_1_2 reference: '' - 90791114-21bb-4a0e-8fc9-5242aa57b3d4: !Template - answer_choices: null - id: 90791114-21bb-4a0e-8fc9-5242aa57b3d4 + bd9bd837-7468-4e93-a022-f2baca56cd62: !Template + answer_choices: A ||| B + id: bd9bd837-7468-4e93-a022-f2baca56cd62 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -153,11 +201,27 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: A/B choice randomized + reference: '' + ce9ba367-5943-4be0-8768-7a2f739d3715: !Template + answer_choices: Yes ||| No + id: ce9ba367-5943-4be0-8768-7a2f739d3715 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences reference: '' - ad82524e-20ad-4114-a372-77474edfe14f: !Template + d31d78e4-2de8-4ca1-aecf-6f22a2507ed9: !Template answer_choices: null - id: ad82524e-20ad-4114-a372-77474edfe14f + id: d31d78e4-2de8-4ca1-aecf-6f22a2507ed9 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer A or B. @@ -194,9 +258,9 @@ templates: original_task: true name: random_choice_A_B_2 reference: '' - b05426a4-6323-43a2-a67a-85f3a15731c7: !Template + d838ecfc-afac-4e9a-b28c-10661d610cec: !Template answer_choices: null - id: b05426a4-6323-43a2-a67a-85f3a15731c7 + id: d838ecfc-afac-4e9a-b28c-10661d610cec jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. diff --git a/promptsource/templates/blimp/npi_present_2/templates.yaml b/promptsource/templates/blimp/npi_present_2/templates.yaml index 48c61629d..fc144f35f 100644 --- a/promptsource/templates/blimp/npi_present_2/templates.yaml +++ b/promptsource/templates/blimp/npi_present_2/templates.yaml @@ -1,48 +1,24 @@ dataset: blimp subset: npi_present_2 templates: - 0ae337be-dac3-41a7-a9c0-96e702a01503: !Template - answer_choices: Yes ||| No - id: 0ae337be-dac3-41a7-a9c0-96e702a01503 - jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] - }} - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences - reference: '' - 169fc98d-bd90-4b90-8e90-703372dfdab8: !Template - answer_choices: Yes ||| No - id: 169fc98d-bd90-4b90-8e90-703372dfdab8 - jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] - }} - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences - reference: '' - 2c7a4a20-601b-47a7-af37-66d83e66cdba: !Template + 09929273-9ae4-4470-a164-d4c3cec38d97: !Template answer_choices: null - id: 2c7a4a20-601b-47a7-af37-66d83e66cdba + id: 09929273-9ae4-4470-a164-d4c3cec38d97 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -50,11 +26,11 @@ templates: {% if shuffled_order == 0 %} - 1 + A {% else %} - 2 + B {% endif %}' metadata: !TemplateMetadata @@ -62,50 +38,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 - reference: '' - 2cdc825c-ca4d-4758-9cfd-a20bfac9aef6: !Template - answer_choices: A ||| B - id: 2cdc825c-ca4d-4758-9cfd-a20bfac9aef6 - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'A/B choice 2 (bad-good, answer: B)' - reference: '' - 70b7fcdb-9e93-411f-80c6-48ad42daf1b0: !Template - answer_choices: B ||| A - id: 70b7fcdb-9e93-411f-80c6-48ad42daf1b0 - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'B/A choice 1 (good-bad, answer: A)' - reference: '' - 88beaf5b-223c-4449-8267-82624ce983c7: !Template - answer_choices: B ||| A - id: 88beaf5b-223c-4449-8267-82624ce983c7 - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'B/A choice, 2 (bad-good, answer: B)' + name: random_choice_A_B reference: '' - 95d9a3b8-722a-47b5-add4-5ecc1f9bf65b: !Template + 32e09079-c0f9-4e4a-8ad6-fb38bc7960e8: !Template answer_choices: null - id: 95d9a3b8-722a-47b5-add4-5ecc1f9bf65b + id: 32e09079-c0f9-4e4a-8ad6-fb38bc7960e8 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer A or B. @@ -142,22 +79,63 @@ templates: original_task: true name: random_choice_A_B_2 reference: '' - b2e28836-c158-4436-bf6b-44d8eddd8dcb: !Template + 4367332e-8220-4ca8-8300-da4ee43ab392: !Template answer_choices: A ||| B - id: b2e28836-c158-4436-bf6b-44d8eddd8dcb - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] - }}' + id: 4367332e-8220-4ca8-8300-da4ee43ab392 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'A/B choice 1 (good-bad, answer: A)' + name: A/B choice randomized (choice order reversed) + reference: '' + 572e0b15-8fbe-41af-8630-1f94d5a40449: !Template + answer_choices: Yes ||| No + id: 572e0b15-8fbe-41af-8630-1f94d5a40449 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences (choice order reversed) reference: '' - e33a6c12-f64b-4a2e-bffe-c67ac4248ca7: !Template + 77f29518-5934-4aa6-a85c-5f99f99ee644: !Template answer_choices: null - id: e33a6c12-f64b-4a2e-bffe-c67ac4248ca7 + id: 77f29518-5934-4aa6-a85c-5f99f99ee644 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer 1 or 2. @@ -194,24 +172,110 @@ templates: original_task: true name: random_choice_1_2 reference: '' - f9aa6f26-0a21-41a4-8f46-b107647c7689: !Template + 78dd7c42-3402-4ba9-ab13-91ff6c5091fe: !Template + answer_choices: A ||| B + id: 78dd7c42-3402-4ba9-ab13-91ff6c5091fe + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: A/B choice randomized + reference: '' + 9176e7fb-1ded-424a-8263-0f5d67a6e6f3: !Template + answer_choices: Yes ||| No + id: 9176e7fb-1ded-424a-8263-0f5d67a6e6f3 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' + b6abf882-a346-4a35-bf34-2067a3a6e9f3: !Template + answer_choices: Yes ||| No + id: b6abf882-a346-4a35-bf34-2067a3a6e9f3 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' + be59edae-2581-41e6-ac77-294704c93ce2: !Template + answer_choices: Yes ||| No + id: be59edae-2581-41e6-ac77-294704c93ce2 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences (choice order reversed) + reference: '' + ef42acf8-084f-49ce-ad22-a5fc48a67d6d: !Template answer_choices: null - id: f9aa6f26-0a21-41a4-8f46-b107647c7689 + id: ef42acf8-084f-49ce-ad22-a5fc48a67d6d jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -219,11 +283,11 @@ templates: {% if shuffled_order == 0 %} - A + 1 {% else %} - B + 2 {% endif %}' metadata: !TemplateMetadata @@ -231,5 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: random_choice_1_2_2 reference: '' diff --git a/promptsource/templates/blimp/only_npi_licensor_present/templates.yaml b/promptsource/templates/blimp/only_npi_licensor_present/templates.yaml index 48ad5fe41..b030475fb 100644 --- a/promptsource/templates/blimp/only_npi_licensor_present/templates.yaml +++ b/promptsource/templates/blimp/only_npi_licensor_present/templates.yaml @@ -1,49 +1,83 @@ dataset: blimp subset: only_npi_licensor_present templates: - 2b0af143-b265-4f52-81df-ea1fe567628d: !Template - answer_choices: A ||| B - id: 2b0af143-b265-4f52-81df-ea1fe567628d - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] - }}' + 0fe83ac3-77bb-4286-8e7c-0d424adef1ab: !Template + answer_choices: Yes ||| No + id: 0fe83ac3-77bb-4286-8e7c-0d424adef1ab + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'A/B choice 2 (bad-good, answer: B)' + original_task: false + name: Yes/No binary judgment for good sentences (choice order reversed) reference: '' - 66751e2b-058d-49ce-b3a9-0089ffe1c019: !Template - answer_choices: null - id: 66751e2b-058d-49ce-b3a9-0089ffe1c019 + 17090f38-019d-46c0-9b26-47f40aee76c4: !Template + answer_choices: Yes ||| No + id: 17090f38-019d-46c0-9b26-47f40aee76c4 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' + 4c89b8cb-1a0c-4e09-996f-e417e3acc286: !Template + answer_choices: Yes ||| No + id: 4c89b8cb-1a0c-4e09-996f-e417e3acc286 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences (choice order reversed) + reference: '' + 62e50096-d057-4c40-baa6-eda3b9a5a6b4: !Template + answer_choices: A ||| B + id: 62e50096-d057-4c40-baa6-eda3b9a5a6b4 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} - {% if shuffled_order == 0 %} - - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -51,51 +85,80 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: A/B choice randomized (choice order reversed) reference: '' - 74011910-24b0-44fd-b4ef-f11dc338257e: !Template - answer_choices: B ||| A - id: 74011910-24b0-44fd-b4ef-f11dc338257e - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] - }}' + 924c661d-6a2c-4fa7-bcb9-c28a913a191d: !Template + answer_choices: Yes ||| No + id: 924c661d-6a2c-4fa7-bcb9-c28a913a191d + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'B/A choice 1 (good-bad, answer: A)' + original_task: false + name: Yes/No binary judgment for bad sentences reference: '' - a4a13d4f-9e72-432c-b43a-a0d9efe9c5c3: !Template - answer_choices: Yes ||| No - id: a4a13d4f-9e72-432c-b43a-a0d9efe9c5c3 - jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] - }} + 98451869-5e77-423b-8e37-57f82450fa81: !Template + answer_choices: A ||| B + id: 98451869-5e77-423b-8e37-57f82450fa81 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata - choices_in_prompt: false + choices_in_prompt: true metrics: - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences + original_task: true + name: A/B choice randomized reference: '' - aa839e00-65d2-4c07-afcd-9abfcfe010ff: !Template + b0265345-5825-43e6-8e75-94b11028d938: !Template answer_choices: null - id: aa839e00-65d2-4c07-afcd-9abfcfe010ff + id: b0265345-5825-43e6-8e75-94b11028d938 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -103,11 +166,11 @@ templates: {% if shuffled_order == 0 %} - A + 1 {% else %} - B + 2 {% endif %}' metadata: !TemplateMetadata @@ -115,14 +178,14 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: random_choice_1_2_2 reference: '' - b7e66774-2b87-40d4-ae17-38ebeec34539: !Template + bc8f2d54-6e71-4be9-8d2c-133254e56c55: !Template answer_choices: null - id: b7e66774-2b87-40d4-ae17-38ebeec34539 + id: bc8f2d54-6e71-4be9-8d2c-133254e56c55 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -154,52 +217,14 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 - reference: '' - e0922693-1a3d-444e-b0da-7e21d6349826: !Template - answer_choices: A ||| B - id: e0922693-1a3d-444e-b0da-7e21d6349826 - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'A/B choice 1 (good-bad, answer: A)' - reference: '' - e117359f-4d81-407d-a8fd-ecba96ccb2ad: !Template - answer_choices: B ||| A - id: e117359f-4d81-407d-a8fd-ecba96ccb2ad - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'B/A choice, 2 (bad-good, answer: B)' - reference: '' - e72f656b-f91d-4147-a2f9-5e6f81f27a41: !Template - answer_choices: Yes ||| No - id: e72f656b-f91d-4147-a2f9-5e6f81f27a41 - jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] - }} - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences + name: random_choice_A_B reference: '' - fb777ae8-278d-447f-a291-a0b76d606f05: !Template + bfbb377d-86a6-4c2c-a603-d9d39a81ca81: !Template answer_choices: null - id: fb777ae8-278d-447f-a291-a0b76d606f05 + id: bfbb377d-86a6-4c2c-a603-d9d39a81ca81 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -231,5 +256,44 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: random_choice_1_2 + reference: '' + d7d2ac5d-101d-4ad2-ac5d-fa414ac7f502: !Template + answer_choices: null + id: d7d2ac5d-101d-4ad2-ac5d-fa414ac7f502 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B_2 reference: '' diff --git a/promptsource/templates/blimp/only_npi_scope/templates.yaml b/promptsource/templates/blimp/only_npi_scope/templates.yaml index b325f45d0..2c4b67341 100644 --- a/promptsource/templates/blimp/only_npi_scope/templates.yaml +++ b/promptsource/templates/blimp/only_npi_scope/templates.yaml @@ -1,74 +1,67 @@ dataset: blimp subset: only_npi_scope templates: - 0a230e5d-c5f6-44b6-9657-2d0f058b1cc9: !Template - answer_choices: B ||| A - id: 0a230e5d-c5f6-44b6-9657-2d0f058b1cc9 - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] - }}' + 004f0c42-5bb2-4695-9b63-90edf2264748: !Template + answer_choices: Yes ||| No + id: 004f0c42-5bb2-4695-9b63-90edf2264748 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'B/A choice, 2 (bad-good, answer: B)' + original_task: false + name: Yes/No binary judgment for good sentences reference: '' - 4d9457ba-b135-4733-88a0-07424781e528: !Template + 105286db-c912-4bae-bfd0-1f5e1a560173: !Template answer_choices: Yes ||| No - id: 4d9457ba-b135-4733-88a0-07424781e528 - jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] - }} + id: 105286db-c912-4bae-bfd0-1f5e1a560173 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata - choices_in_prompt: false + choices_in_prompt: true metrics: - Accuracy original_task: false - name: Yes/No binary judgment for bad sentences + name: Yes/No binary judgment for good sentences (choice order reversed) reference: '' - 64de70aa-ecac-4a2c-8c74-da2347a3dfe1: !Template + 39ebdce9-997a-4c40-aeab-1e6dc35c1e3b: !Template answer_choices: A ||| B - id: 64de70aa-ecac-4a2c-8c74-da2347a3dfe1 - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'A/B choice 1 (good-bad, answer: A)' - reference: '' - 7aed9ddb-0797-4ce8-9669-ae7e9fe96416: !Template - answer_choices: null - id: 7aed9ddb-0797-4ce8-9669-ae7e9fe96416 + id: 39ebdce9-997a-4c40-aeab-1e6dc35c1e3b jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -76,11 +69,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: A/B choice randomized (choice order reversed) reference: '' - 935da619-659d-4a9f-84ff-3701a43c2eac: !Template + 3dceb71f-4e37-4ed9-a4f8-ad5c442fa7a8: !Template answer_choices: null - id: 935da619-659d-4a9f-84ff-3701a43c2eac + id: 3dceb71f-4e37-4ed9-a4f8-ad5c442fa7a8 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer 1 or 2. @@ -117,47 +110,47 @@ templates: original_task: true name: random_choice_1_2 reference: '' - 94b50572-2a21-450b-96d8-24ab3a9e8edf: !Template - answer_choices: Yes ||| No - id: 94b50572-2a21-450b-96d8-24ab3a9e8edf - jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] - }} - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences - reference: '' - a0ae522c-bf49-49c3-8acd-a32a4f7c8086: !Template + 5e04d59a-2491-4990-893c-eb8ad7d0c96a: !Template answer_choices: A ||| B - id: a0ae522c-bf49-49c3-8acd-a32a4f7c8086 - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'A/B choice 2 (bad-good, answer: B)' - reference: '' - b53108b5-95a3-4b3e-99ee-d9f4dc9c0358: !Template - answer_choices: B ||| A - id: b53108b5-95a3-4b3e-99ee-d9f4dc9c0358 - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] - }}' + id: 5e04d59a-2491-4990-893c-eb8ad7d0c96a + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'B/A choice 1 (good-bad, answer: A)' + name: A/B choice randomized reference: '' - b8c83d50-3d52-4f17-a173-f9a1588bcb25: !Template + 8b450990-2efb-48e1-a903-4d5837951de2: !Template answer_choices: null - id: b8c83d50-3d52-4f17-a173-f9a1588bcb25 + id: 8b450990-2efb-48e1-a903-4d5837951de2 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer A or B. @@ -194,9 +187,9 @@ templates: original_task: true name: random_choice_A_B reference: '' - cc03dab5-2b6f-4718-9b06-063a4cf3891f: !Template + a4e614f0-7bb4-4723-86b4-3579f86a50a5: !Template answer_choices: null - id: cc03dab5-2b6f-4718-9b06-063a4cf3891f + id: a4e614f0-7bb4-4723-86b4-3579f86a50a5 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer A or B. @@ -233,3 +226,74 @@ templates: original_task: true name: random_choice_A_B_2 reference: '' + c68b76db-1764-410d-b3cc-09c5cf0d3ebd: !Template + answer_choices: null + id: c68b76db-1764-410d-b3cc-09c5cf0d3ebd + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2_2 + reference: '' + d903517e-dfec-43e1-880e-c8f4a48da0c0: !Template + answer_choices: Yes ||| No + id: d903517e-dfec-43e1-880e-c8f4a48da0c0 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' + e11d4e8e-8451-447e-be52-9848028c57d1: !Template + answer_choices: Yes ||| No + id: e11d4e8e-8451-447e-be52-9848028c57d1 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences (choice order reversed) + reference: '' diff --git a/promptsource/templates/blimp/passive_1/templates.yaml b/promptsource/templates/blimp/passive_1/templates.yaml index f52ba25aa..a75578a75 100644 --- a/promptsource/templates/blimp/passive_1/templates.yaml +++ b/promptsource/templates/blimp/passive_1/templates.yaml @@ -1,12 +1,12 @@ dataset: blimp subset: passive_1 templates: - 009eade8-706c-4d35-a3c4-aa1d7abf1c44: !Template + 2f0cfc41-575a-4ac5-8d63-e5ac46b247de: !Template answer_choices: null - id: 009eade8-706c-4d35-a3c4-aa1d7abf1c44 + id: 2f0cfc41-575a-4ac5-8d63-e5ac46b247de jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -38,65 +38,96 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: random_choice_A_B_2 reference: '' - 423ae75a-bf5b-432d-a7b1-886787b8e042: !Template - answer_choices: B ||| A - id: 423ae75a-bf5b-432d-a7b1-886787b8e042 - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] - }}' + 62971a37-542c-4c48-bc1b-d9f5d3801f25: !Template + answer_choices: Yes ||| No + id: 62971a37-542c-4c48-bc1b-d9f5d3801f25 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'B/A choice 1 (good-bad, answer: A)' + original_task: false + name: Yes/No binary judgment for good sentences (choice order reversed) reference: '' - 6e5547b8-ee1d-4c88-8710-8d0350c0fec4: !Template - answer_choices: B ||| A - id: 6e5547b8-ee1d-4c88-8710-8d0350c0fec4 - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] - }}' + 63b3fd0a-759e-44e2-91fe-ba7bf515813c: !Template + answer_choices: A ||| B + id: 63b3fd0a-759e-44e2-91fe-ba7bf515813c + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'B/A choice, 2 (bad-good, answer: B)' + name: A/B choice randomized reference: '' - 880be2b4-7fd4-482f-81e7-06224b2348b3: !Template - answer_choices: A ||| B - id: 880be2b4-7fd4-482f-81e7-06224b2348b3 - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] - }}' + 711f6d10-413a-4745-8afd-a0e30588eec8: !Template + answer_choices: Yes ||| No + id: 711f6d10-413a-4745-8afd-a0e30588eec8 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'A/B choice 1 (good-bad, answer: A)' + original_task: false + name: Yes/No binary judgment for good sentences reference: '' - 96f50bba-2d1c-45f8-bc4d-4e525861070f: !Template + 8d5ab8c2-948f-4a38-9fda-d6c9e142f147: !Template answer_choices: null - id: 96f50bba-2d1c-45f8-bc4d-4e525861070f + id: 8d5ab8c2-948f-4a38-9fda-d6c9e142f147 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -104,11 +135,11 @@ templates: {% if shuffled_order == 0 %} - 1 + A {% else %} - 2 + B {% endif %}' metadata: !TemplateMetadata @@ -116,26 +147,68 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: random_choice_A_B reference: '' - 9a67d7ba-dc0d-4365-adc0-ff49fcfc0d0c: !Template + 9be7454f-55aa-49b7-a30f-7c95d3311162: !Template answer_choices: Yes ||| No - id: 9a67d7ba-dc0d-4365-adc0-ff49fcfc0d0c - jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] - }} + id: 9be7454f-55aa-49b7-a30f-7c95d3311162 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata - choices_in_prompt: false + choices_in_prompt: true metrics: - Accuracy original_task: false - name: Yes/No binary judgment for bad sentences + name: Yes/No binary judgment for bad sentences (choice order reversed) + reference: '' + a779ca82-98c2-4bae-a29f-9bf61f402166: !Template + answer_choices: A ||| B + id: a779ca82-98c2-4bae-a29f-9bf61f402166 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: A/B choice randomized (choice order reversed) reference: '' - b1f1f897-fd46-4862-bf43-dea47271b37f: !Template + c7ade213-fc39-4881-b2f5-2bf240851c52: !Template answer_choices: null - id: b1f1f897-fd46-4862-bf43-dea47271b37f + id: c7ade213-fc39-4881-b2f5-2bf240851c52 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -167,51 +240,42 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: random_choice_1_2 reference: '' - c3d01433-ddba-4b7c-bd60-18cef6237a42: !Template + ca4a66a8-ccbb-422b-b1d3-dc1cfef1c560: !Template answer_choices: Yes ||| No - id: c3d01433-ddba-4b7c-bd60-18cef6237a42 - jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] - }} - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences - reference: '' - db7ddd1e-3f66-414f-9b02-2254154a6103: !Template - answer_choices: A ||| B - id: db7ddd1e-3f66-414f-9b02-2254154a6103 - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] - }}' + id: ca4a66a8-ccbb-422b-b1d3-dc1cfef1c560 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'A/B choice 2 (bad-good, answer: B)' + original_task: false + name: Yes/No binary judgment for bad sentences reference: '' - f28e549e-34bd-4a7f-af29-570dd110cac0: !Template + fa74b05a-f043-4703-90ca-b55f7857450a: !Template answer_choices: null - id: f28e549e-34bd-4a7f-af29-570dd110cac0 + id: fa74b05a-f043-4703-90ca-b55f7857450a jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -219,11 +283,11 @@ templates: {% if shuffled_order == 0 %} - A + 1 {% else %} - B + 2 {% endif %}' metadata: !TemplateMetadata @@ -231,5 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: random_choice_1_2_2 reference: '' diff --git a/promptsource/templates/blimp/passive_2/templates.yaml b/promptsource/templates/blimp/passive_2/templates.yaml index fe0c2db2e..f4cacc498 100644 --- a/promptsource/templates/blimp/passive_2/templates.yaml +++ b/promptsource/templates/blimp/passive_2/templates.yaml @@ -1,49 +1,35 @@ dataset: blimp subset: passive_2 templates: - 0d59ed08-a317-4672-ade2-3ef1ef4eda64: !Template - answer_choices: B ||| A - id: 0d59ed08-a317-4672-ade2-3ef1ef4eda64 - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'B/A choice 1 (good-bad, answer: A)' - reference: '' - 196bba57-312d-4c64-b56d-ce1dac736772: !Template - answer_choices: null - id: 196bba57-312d-4c64-b56d-ce1dac736772 + 0db23227-68fd-45ba-9657-a780db36aa2a: !Template + answer_choices: A ||| B + id: 0db23227-68fd-45ba-9657-a780db36aa2a jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -51,11 +37,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: A/B choice randomized reference: '' - 2ecc2dca-9b3d-4340-85ec-9ed8ae25714a: !Template + 2b8e7e36-b410-4615-a821-43b9e24d301d: !Template answer_choices: null - id: 2ecc2dca-9b3d-4340-85ec-9ed8ae25714a + id: 2b8e7e36-b410-4615-a821-43b9e24d301d jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer A or B. @@ -92,87 +78,150 @@ templates: original_task: true name: random_choice_A_B_2 reference: '' - 3cc44323-551c-464d-8222-dbc8dcad1873: !Template - answer_choices: A ||| B - id: 3cc44323-551c-464d-8222-dbc8dcad1873 - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] - }}' + 4ac2289f-b34c-4795-8541-f5d6d8688950: !Template + answer_choices: Yes ||| No + id: 4ac2289f-b34c-4795-8541-f5d6d8688950 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'A/B choice 2 (bad-good, answer: B)' + original_task: false + name: Yes/No binary judgment for good sentences reference: '' - 58684b10-f0a2-484c-8bb3-9f565623474a: !Template - answer_choices: B ||| A - id: 58684b10-f0a2-484c-8bb3-9f565623474a - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] - }}' + 6a5ae41a-3d8f-4733-b158-734c1d0538b7: !Template + answer_choices: null + id: 6a5ae41a-3d8f-4733-b158-734c1d0538b7 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'B/A choice, 2 (bad-good, answer: B)' + name: random_choice_A_B reference: '' - 675e20e3-aabe-4254-9df0-a3b24b2f4adf: !Template - answer_choices: A ||| B - id: 675e20e3-aabe-4254-9df0-a3b24b2f4adf - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] - }}' + 72e7d5c3-aca0-4851-a0fd-e0c275c67c81: !Template + answer_choices: null + id: 72e7d5c3-aca0-4851-a0fd-e0c275c67c81 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'A/B choice 1 (good-bad, answer: A)' + name: random_choice_1_2 reference: '' - 69d66f1a-eaed-40da-b847-3e90e61b1418: !Template + 8605bea0-1e0c-478c-8527-76fbc3dbb894: !Template answer_choices: Yes ||| No - id: 69d66f1a-eaed-40da-b847-3e90e61b1418 - jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] - }} + id: 8605bea0-1e0c-478c-8527-76fbc3dbb894 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata - choices_in_prompt: false + choices_in_prompt: true metrics: - Accuracy original_task: false - name: Yes/No binary judgment for good sentences + name: Yes/No binary judgment for bad sentences (choice order reversed) reference: '' - a1e17a9c-ad8a-4dae-a022-161084f75a97: !Template + 8df0b50c-9efd-4dd2-b660-6a912d1d9f43: !Template answer_choices: Yes ||| No - id: a1e17a9c-ad8a-4dae-a022-161084f75a97 - jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] - }} + id: 8df0b50c-9efd-4dd2-b660-6a912d1d9f43 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata - choices_in_prompt: false + choices_in_prompt: true metrics: - Accuracy original_task: false name: Yes/No binary judgment for bad sentences reference: '' - c87bad0e-bf1f-4f5a-9e82-9f78a83493df: !Template + 9ef44939-488f-4a2a-9670-fd85750584f7: !Template answer_choices: null - id: c87bad0e-bf1f-4f5a-9e82-9f78a83493df + id: 9ef44939-488f-4a2a-9670-fd85750584f7 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -180,11 +229,11 @@ templates: {% if shuffled_order == 0 %} - A + 1 {% else %} - B + 2 {% endif %}' metadata: !TemplateMetadata @@ -192,38 +241,53 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: random_choice_1_2_2 reference: '' - d7cc6c34-590e-4f31-8a4c-9ddb9ceb824b: !Template - answer_choices: null - id: d7cc6c34-590e-4f31-8a4c-9ddb9ceb824b + e84d0c9e-ba07-464f-9ffb-b762864a5db6: !Template + answer_choices: Yes ||| No + id: e84d0c9e-ba07-464f-9ffb-b762864a5db6 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences (choice order reversed) + reference: '' + fdef417a-2746-4e96-ab73-cc6fcd748d57: !Template + answer_choices: A ||| B + id: fdef417a-2746-4e96-ab73-cc6fcd748d57 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -231,5 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: A/B choice randomized (choice order reversed) reference: '' diff --git a/promptsource/templates/blimp/principle_A_c_command/templates.yaml b/promptsource/templates/blimp/principle_A_c_command/templates.yaml index 3a55cd532..debbc30e3 100644 --- a/promptsource/templates/blimp/principle_A_c_command/templates.yaml +++ b/promptsource/templates/blimp/principle_A_c_command/templates.yaml @@ -1,36 +1,35 @@ dataset: blimp subset: principle_A_c_command templates: - 005e713f-7db1-4e6c-8eff-9d4b430150de: !Template - answer_choices: null - id: 005e713f-7db1-4e6c-8eff-9d4b430150de + 2baf720f-f276-4453-bc65-aa1243db1628: !Template + answer_choices: A ||| B + id: 2baf720f-f276-4453-bc65-aa1243db1628 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} - {% if shuffled_order == 0 %} - - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -38,38 +37,42 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: A/B choice randomized reference: '' - 17a6b0f6-e715-489f-9ff3-67b28778d152: !Template + 3ccc7b98-a1f0-40d0-887f-6b8b4d02be85: !Template answer_choices: Yes ||| No - id: 17a6b0f6-e715-489f-9ff3-67b28778d152 - jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] - }} + id: 3ccc7b98-a1f0-40d0-887f-6b8b4d02be85 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata - choices_in_prompt: false + choices_in_prompt: true metrics: - Accuracy original_task: false - name: Yes/No binary judgment for bad sentences + name: Yes/No binary judgment for bad sentences (choice order reversed) reference: '' - 2d5b5be8-da66-4716-91fc-4a593dc35ec4: !Template + 52726eee-092b-45c1-92c7-5846729ad713: !Template answer_choices: null - id: 2d5b5be8-da66-4716-91fc-4a593dc35ec4 + id: 52726eee-092b-45c1-92c7-5846729ad713 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -77,11 +80,11 @@ templates: {% if shuffled_order == 0 %} - 1 + A {% else %} - 2 + B {% endif %}' metadata: !TemplateMetadata @@ -89,37 +92,81 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: random_choice_A_B reference: '' - 4c14aacc-78a7-4afb-b3b9-69da2a9d1d64: !Template - answer_choices: B ||| A - id: 4c14aacc-78a7-4afb-b3b9-69da2a9d1d64 - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] - }}' + 5b929665-8a28-404d-b199-badc6d460dfc: !Template + answer_choices: Yes ||| No + id: 5b929665-8a28-404d-b199-badc6d460dfc + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'B/A choice, 2 (bad-good, answer: B)' + original_task: false + name: Yes/No binary judgment for bad sentences reference: '' - 54480a25-a63a-476c-a3cf-edbf9c92278f: !Template + 7170e0c2-e190-4749-acc9-8728a24e2ead: !Template + answer_choices: Yes ||| No + id: 7170e0c2-e190-4749-acc9-8728a24e2ead + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences (choice order reversed) + reference: '' + 81723db6-ba6e-44c8-9ac1-bcdf85e6fd4d: !Template answer_choices: A ||| B - id: 54480a25-a63a-476c-a3cf-edbf9c92278f - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] - }}' + id: 81723db6-ba6e-44c8-9ac1-bcdf85e6fd4d + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'A/B choice 2 (bad-good, answer: B)' + name: A/B choice randomized (choice order reversed) reference: '' - 65b6ca78-d43c-4350-a03c-fd66c0379a79: !Template + 84c688a8-d189-472f-bce2-3086b989f9a7: !Template answer_choices: null - id: 65b6ca78-d43c-4350-a03c-fd66c0379a79 + id: 84c688a8-d189-472f-bce2-3086b989f9a7 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -156,37 +203,67 @@ templates: original_task: true name: random_choice_1_2_2 reference: '' - 7e773aac-5edf-488b-82a1-1c98b906091d: !Template - answer_choices: A ||| B - id: 7e773aac-5edf-488b-82a1-1c98b906091d - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] - }}' + 8d7af12a-6b0a-47cf-aafa-09d9f9c4adf2: !Template + answer_choices: Yes ||| No + id: 8d7af12a-6b0a-47cf-aafa-09d9f9c4adf2 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'A/B choice 1 (good-bad, answer: A)' + original_task: false + name: Yes/No binary judgment for good sentences reference: '' - a5059bc2-b979-485f-8935-ade9466030e8: !Template - answer_choices: Yes ||| No - id: a5059bc2-b979-485f-8935-ade9466030e8 - jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] - }} + 9c5d4d2c-89fe-4536-98a6-40d6d406ef0b: !Template + answer_choices: null + id: 9c5d4d2c-89fe-4536-98a6-40d6d406ef0b + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' metadata: !TemplateMetadata - choices_in_prompt: false + choices_in_prompt: true metrics: - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences + original_task: true + name: random_choice_1_2 reference: '' - bc47b240-80c8-4c88-a506-8e22fc58af1d: !Template + cf193f1f-4c5c-4472-b5c3-bbdd0840c41c: !Template answer_choices: null - id: bc47b240-80c8-4c88-a506-8e22fc58af1d + id: cf193f1f-4c5c-4472-b5c3-bbdd0840c41c jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -218,18 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B - reference: '' - fd96b228-90d9-4c99-bb2e-301b63860104: !Template - answer_choices: B ||| A - id: fd96b228-90d9-4c99-bb2e-301b63860104 - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'B/A choice 1 (good-bad, answer: A)' + name: random_choice_A_B_2 reference: '' diff --git a/promptsource/templates/blimp/principle_A_case_1/templates.yaml b/promptsource/templates/blimp/principle_A_case_1/templates.yaml index 561ccd940..aaaaeb277 100644 --- a/promptsource/templates/blimp/principle_A_case_1/templates.yaml +++ b/promptsource/templates/blimp/principle_A_case_1/templates.yaml @@ -1,36 +1,24 @@ dataset: blimp subset: principle_A_case_1 templates: - 0ff388ad-d085-4f3f-ba26-384408db607b: !Template - answer_choices: Yes ||| No - id: 0ff388ad-d085-4f3f-ba26-384408db607b - jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] - }} - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences - reference: '' - 1006fde0-2bd9-4432-bba2-4edfb104f733: !Template + 015dcbc0-3c03-4567-954e-e827cb378af5: !Template answer_choices: null - id: 1006fde0-2bd9-4432-bba2-4edfb104f733 + id: 015dcbc0-3c03-4567-954e-e827cb378af5 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -38,11 +26,11 @@ templates: {% if shuffled_order == 0 %} - 1 + A {% else %} - 2 + B {% endif %}' metadata: !TemplateMetadata @@ -50,39 +38,42 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: random_choice_A_B reference: '' - 350577af-e811-4978-9202-9010b6bbfc72: !Template - answer_choices: B ||| A - id: 350577af-e811-4978-9202-9010b6bbfc72 - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] - }}' + 36eb9af7-58d0-4015-afdd-7368bed1c976: !Template + answer_choices: Yes ||| No + id: 36eb9af7-58d0-4015-afdd-7368bed1c976 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'B/A choice 1 (good-bad, answer: A)' + original_task: false + name: Yes/No binary judgment for bad sentences reference: '' - 61dfec5f-81e1-4932-a9de-a16928c0908d: !Template + 45e98aed-70ef-40b5-b632-3bdda7691c7e: !Template answer_choices: null - id: 61dfec5f-81e1-4932-a9de-a16928c0908d + id: 45e98aed-70ef-40b5-b632-3bdda7691c7e jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -90,11 +81,11 @@ templates: {% if shuffled_order == 0 %} - A + 1 {% else %} - B + 2 {% endif %}' metadata: !TemplateMetadata @@ -102,14 +93,14 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: random_choice_1_2_2 reference: '' - 6849da5f-97d5-4478-a8f5-296f59698925: !Template + 6720534d-1a15-49e9-b7f1-63a6afe34ea0: !Template answer_choices: null - id: 6849da5f-97d5-4478-a8f5-296f59698925 + id: 6720534d-1a15-49e9-b7f1-63a6afe34ea0 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -141,38 +132,37 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: random_choice_1_2 reference: '' - 9140207f-a54a-45fb-a239-823c4370ac8d: !Template - answer_choices: null - id: 9140207f-a54a-45fb-a239-823c4370ac8d + 733c3230-e0eb-479d-bfd6-bfc60651076d: !Template + answer_choices: A ||| B + id: 733c3230-e0eb-479d-bfd6-bfc60651076d jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} - {% if shuffled_order == 0 %} - - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -180,56 +170,130 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: A/B choice randomized reference: '' - bd84633c-bc72-4a34-be17-624572b6ac58: !Template - answer_choices: A ||| B - id: bd84633c-bc72-4a34-be17-624572b6ac58 - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] - }}' + 998c717c-041a-4a39-890f-00dd33d923dc: !Template + answer_choices: Yes ||| No + id: 998c717c-041a-4a39-890f-00dd33d923dc + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'A/B choice 1 (good-bad, answer: A)' + original_task: false + name: Yes/No binary judgment for bad sentences (choice order reversed) reference: '' - c00c09b8-9962-423f-9778-2af62b981baa: !Template + 9cfdebd3-c056-4a49-b479-86c259a4b349: !Template answer_choices: A ||| B - id: c00c09b8-9962-423f-9778-2af62b981baa - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] - }}' + id: 9cfdebd3-c056-4a49-b479-86c259a4b349 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'A/B choice 2 (bad-good, answer: B)' + name: A/B choice randomized (choice order reversed) + reference: '' + c42a7b10-1ec8-4fec-aeac-c3803047f2df: !Template + answer_choices: Yes ||| No + id: c42a7b10-1ec8-4fec-aeac-c3803047f2df + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences (choice order reversed) reference: '' - cc2ebe4a-51f8-4c8c-998d-3104ed3663f6: !Template + f8337f64-b45d-4e1a-8523-f6ec734e942d: !Template answer_choices: Yes ||| No - id: cc2ebe4a-51f8-4c8c-998d-3104ed3663f6 - jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] - }} + id: f8337f64-b45d-4e1a-8523-f6ec734e942d + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata - choices_in_prompt: false + choices_in_prompt: true metrics: - Accuracy original_task: false name: Yes/No binary judgment for good sentences reference: '' - f0479c50-3f02-4881-9e14-a05eaef12181: !Template - answer_choices: B ||| A - id: f0479c50-3f02-4881-9e14-a05eaef12181 - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] - }}' + fc4314a3-ec10-4d15-8d66-fd6b8b02b7b9: !Template + answer_choices: null + id: fc4314a3-ec10-4d15-8d66-fd6b8b02b7b9 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'B/A choice, 2 (bad-good, answer: B)' + name: random_choice_A_B_2 reference: '' diff --git a/promptsource/templates/blimp/principle_A_case_2/templates.yaml b/promptsource/templates/blimp/principle_A_case_2/templates.yaml index e25d20370..8a953353f 100644 --- a/promptsource/templates/blimp/principle_A_case_2/templates.yaml +++ b/promptsource/templates/blimp/principle_A_case_2/templates.yaml @@ -1,24 +1,40 @@ dataset: blimp subset: principle_A_case_2 templates: - 1783c375-fcfa-4792-b00a-6968bf2ed883: !Template + 0e06f97d-bd62-4a61-a8b5-2720938f0c8b: !Template + answer_choices: Yes ||| No + id: 0e06f97d-bd62-4a61-a8b5-2720938f0c8b + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences (choice order reversed) + reference: '' + 36af5545-6402-467f-9910-4afaa4fe8e83: !Template answer_choices: null - id: 1783c375-fcfa-4792-b00a-6968bf2ed883 + id: 36af5545-6402-467f-9910-4afaa4fe8e83 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -26,11 +42,11 @@ templates: {% if shuffled_order == 0 %} - A + 1 {% else %} - B + 2 {% endif %}' metadata: !TemplateMetadata @@ -38,77 +54,107 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: random_choice_1_2 reference: '' - 259259c7-159c-4328-8906-dac38f978b84: !Template + 5f9bb974-505c-4cbd-91dd-aaf624cc9155: !Template answer_choices: A ||| B - id: 259259c7-159c-4328-8906-dac38f978b84 - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'A/B choice 1 (good-bad, answer: A)' - reference: '' - 30723e9b-dce5-43d9-812a-bb71a3dd5a46: !Template - answer_choices: B ||| A - id: 30723e9b-dce5-43d9-812a-bb71a3dd5a46 - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] - }}' + id: 5f9bb974-505c-4cbd-91dd-aaf624cc9155 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'B/A choice, 2 (bad-good, answer: B)' - reference: '' - 6a1b4ed8-80b1-4fce-ad74-4637644ebaba: !Template - answer_choices: Yes ||| No - id: 6a1b4ed8-80b1-4fce-ad74-4637644ebaba - jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] - }} - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences + name: A/B choice randomized (choice order reversed) reference: '' - 73c23d5a-9a8d-4163-bc89-0bd4972e931c: !Template + 645d1908-7f34-4d1b-899c-a7ba3defcc17: !Template answer_choices: Yes ||| No - id: 73c23d5a-9a8d-4163-bc89-0bd4972e931c - jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] - }} + id: 645d1908-7f34-4d1b-899c-a7ba3defcc17 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata - choices_in_prompt: false + choices_in_prompt: true metrics: - Accuracy original_task: false - name: Yes/No binary judgment for bad sentences + name: Yes/No binary judgment for good sentences (choice order reversed) reference: '' - 766aa4a1-42e8-476a-a007-c31bc355f800: !Template - answer_choices: A ||| B - id: 766aa4a1-42e8-476a-a007-c31bc355f800 - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] - }}' + 7f8f8557-5a60-4901-a6d1-18b7a9f543fe: !Template + answer_choices: null + id: 7f8f8557-5a60-4901-a6d1-18b7a9f543fe + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'A/B choice 2 (bad-good, answer: B)' + name: random_choice_1_2_2 reference: '' - 8375447f-7e6c-4baf-a7e4-3be7f333c1fa: !Template + 889961b0-cf52-4659-ad58-5372883bb19f: !Template answer_choices: null - id: 8375447f-7e6c-4baf-a7e4-3be7f333c1fa + id: 889961b0-cf52-4659-ad58-5372883bb19f jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -140,26 +186,26 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: random_choice_A_B_2 reference: '' - b7f484ac-57f1-4e48-ad33-b82bcd5e6d1b: !Template + 8cdda726-0e4c-44f0-a986-793e9bd5d8d2: !Template answer_choices: null - id: b7f484ac-57f1-4e48-ad33-b82bcd5e6d1b + id: 8cdda726-0e4c-44f0-a986-793e9bd5d8d2 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -167,11 +213,11 @@ templates: {% if shuffled_order == 0 %} - 1 + A {% else %} - 2 + B {% endif %}' metadata: !TemplateMetadata @@ -179,38 +225,53 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: random_choice_A_B reference: '' - c7875338-345d-4159-ab5e-7378e077b354: !Template - answer_choices: null - id: c7875338-345d-4159-ab5e-7378e077b354 + 99743879-4158-4cc2-a65a-30b88ae9be63: !Template + answer_choices: Yes ||| No + id: 99743879-4158-4cc2-a65a-30b88ae9be63 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' + b62169dc-11f0-409d-aff7-c4e1c18e5f5f: !Template + answer_choices: A ||| B + id: b62169dc-11f0-409d-aff7-c4e1c18e5f5f jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} - {% if shuffled_order == 0 %} - - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -218,18 +279,21 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: A/B choice randomized reference: '' - f99d7eff-f127-42ea-8bad-d573a9ff7178: !Template - answer_choices: B ||| A - id: f99d7eff-f127-42ea-8bad-d573a9ff7178 - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] - }}' + e204ff36-aed4-4b9c-9f22-2f759e15dfb7: !Template + answer_choices: Yes ||| No + id: e204ff36-aed4-4b9c-9f22-2f759e15dfb7 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'B/A choice 1 (good-bad, answer: A)' + original_task: false + name: Yes/No binary judgment for good sentences reference: '' diff --git a/promptsource/templates/blimp/principle_A_domain_1/templates.yaml b/promptsource/templates/blimp/principle_A_domain_1/templates.yaml index 3f425377a..7b49d7743 100644 --- a/promptsource/templates/blimp/principle_A_domain_1/templates.yaml +++ b/promptsource/templates/blimp/principle_A_domain_1/templates.yaml @@ -1,24 +1,24 @@ dataset: blimp subset: principle_A_domain_1 templates: - 1fd5a2fa-e668-4c8a-9ebe-2957671ad095: !Template + 1eac62d1-f9c0-4937-8161-cdedbfd90b60: !Template answer_choices: null - id: 1fd5a2fa-e668-4c8a-9ebe-2957671ad095 + id: 1eac62d1-f9c0-4937-8161-cdedbfd90b60 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -26,11 +26,11 @@ templates: {% if shuffled_order == 0 %} - 1 + A {% else %} - 2 + B {% endif %}' metadata: !TemplateMetadata @@ -38,52 +38,42 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 - reference: '' - 23d44482-13c3-420a-981b-5ae39c8898f6: !Template - answer_choices: A ||| B - id: 23d44482-13c3-420a-981b-5ae39c8898f6 - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'A/B choice 2 (bad-good, answer: B)' + name: random_choice_A_B reference: '' - 72177185-ab22-4403-8d10-174b680414cf: !Template - answer_choices: B ||| A - id: 72177185-ab22-4403-8d10-174b680414cf - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] - }}' + 5b2541e0-4d4f-4dc1-b551-5484226ff639: !Template + answer_choices: Yes ||| No + id: 5b2541e0-4d4f-4dc1-b551-5484226ff639 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'B/A choice, 2 (bad-good, answer: B)' + original_task: false + name: Yes/No binary judgment for good sentences (choice order reversed) reference: '' - a9f8b70e-a109-4c8d-95b7-78643fe3509e: !Template + 6b667e46-ba6b-472a-beed-4ea311ce13ea: !Template answer_choices: null - id: a9f8b70e-a109-4c8d-95b7-78643fe3509e + id: 6b667e46-ba6b-472a-beed-4ea311ce13ea jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -91,11 +81,11 @@ templates: {% if shuffled_order == 0 %} - 1 + A {% else %} - 2 + B {% endif %}' metadata: !TemplateMetadata @@ -103,26 +93,42 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: random_choice_A_B_2 + reference: '' + 8d398353-b666-4065-8e94-5e4371097553: !Template + answer_choices: Yes ||| No + id: 8d398353-b666-4065-8e94-5e4371097553 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences (choice order reversed) reference: '' - c7447ade-65ac-496b-9e82-0458c177b74f: !Template + 9534d724-9d22-4ddf-ad3e-4e46f23e0749: !Template answer_choices: null - id: c7447ade-65ac-496b-9e82-0458c177b74f + id: 9534d724-9d22-4ddf-ad3e-4e46f23e0749 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -130,11 +136,11 @@ templates: {% if shuffled_order == 0 %} - A + 1 {% else %} - B + 2 {% endif %}' metadata: !TemplateMetadata @@ -142,50 +148,58 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: random_choice_1_2 reference: '' - cdd87a1b-d9c8-44bd-8db3-5974e00d00fe: !Template + 9c95c009-2d1e-45a4-b7c5-37002b708ddd: !Template answer_choices: Yes ||| No - id: cdd87a1b-d9c8-44bd-8db3-5974e00d00fe - jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] - }} + id: 9c95c009-2d1e-45a4-b7c5-37002b708ddd + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata - choices_in_prompt: false + choices_in_prompt: true metrics: - Accuracy original_task: false - name: Yes/No binary judgment for good sentences + name: Yes/No binary judgment for bad sentences reference: '' - d4630f14-86de-44a6-832b-7b71eb2e48b2: !Template + b4ac6f8b-a749-47e7-9296-43677a4f0ab9: !Template answer_choices: Yes ||| No - id: d4630f14-86de-44a6-832b-7b71eb2e48b2 - jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] - }} + id: b4ac6f8b-a749-47e7-9296-43677a4f0ab9 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata - choices_in_prompt: false + choices_in_prompt: true metrics: - Accuracy original_task: false - name: Yes/No binary judgment for bad sentences + name: Yes/No binary judgment for good sentences reference: '' - d4db4dfe-74df-47d9-870d-65cf6461759b: !Template + c1dc8b28-d454-4a66-b377-19d132d7d989: !Template answer_choices: null - id: d4db4dfe-74df-47d9-870d-65cf6461759b + id: c1dc8b28-d454-4a66-b377-19d132d7d989 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -193,11 +207,11 @@ templates: {% if shuffled_order == 0 %} - A + 1 {% else %} - B + 2 {% endif %}' metadata: !TemplateMetadata @@ -205,31 +219,81 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: random_choice_1_2_2 reference: '' - ebf0f263-8113-46fa-902a-f8fb710a0bcd: !Template - answer_choices: B ||| A - id: ebf0f263-8113-46fa-902a-f8fb710a0bcd - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] - }}' + c4083af3-6f77-4007-ac92-094ff365c8cf: !Template + answer_choices: A ||| B + id: c4083af3-6f77-4007-ac92-094ff365c8cf + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'B/A choice 1 (good-bad, answer: A)' + name: A/B choice randomized reference: '' - efc49f08-4b8a-4477-9d51-dfb31a062e56: !Template + e13ab047-478e-4990-bc57-7db558d026a0: !Template answer_choices: A ||| B - id: efc49f08-4b8a-4477-9d51-dfb31a062e56 - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] - }}' + id: e13ab047-478e-4990-bc57-7db558d026a0 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'A/B choice 1 (good-bad, answer: A)' + name: A/B choice randomized (choice order reversed) reference: '' diff --git a/promptsource/templates/blimp/principle_A_domain_2/templates.yaml b/promptsource/templates/blimp/principle_A_domain_2/templates.yaml index 0bbd0d742..3f5911adc 100644 --- a/promptsource/templates/blimp/principle_A_domain_2/templates.yaml +++ b/promptsource/templates/blimp/principle_A_domain_2/templates.yaml @@ -1,25 +1,50 @@ dataset: blimp subset: principle_A_domain_2 templates: - 04f77863-ded0-408a-b930-69b2ede392aa: !Template - answer_choices: B ||| A - id: 04f77863-ded0-408a-b930-69b2ede392aa - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] - }}' + 11974cf5-c168-4764-8274-85fa22cf73e4: !Template + answer_choices: A ||| B + id: 11974cf5-c168-4764-8274-85fa22cf73e4 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'B/A choice 1 (good-bad, answer: A)' + name: A/B choice randomized (choice order reversed) reference: '' - 247b2f62-e586-4edd-9e55-a70a3ba822e3: !Template + 219461e6-ef46-443c-870e-8ec5bf400090: !Template answer_choices: null - id: 247b2f62-e586-4edd-9e55-a70a3ba822e3 + id: 219461e6-ef46-443c-870e-8ec5bf400090 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -51,24 +76,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 - reference: '' - 3c6ef6c4-2fdb-4e67-80c8-ed54ae9fa001: !Template - answer_choices: A ||| B - id: 3c6ef6c4-2fdb-4e67-80c8-ed54ae9fa001 - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'A/B choice 1 (good-bad, answer: A)' + name: random_choice_1_2 reference: '' - 55e19004-f66c-421f-92fc-dcc1f3e27381: !Template + 27659ea3-841d-4d92-a146-784bc8d1cdb4: !Template answer_choices: null - id: 55e19004-f66c-421f-92fc-dcc1f3e27381 + id: 27659ea3-841d-4d92-a146-784bc8d1cdb4 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer A or B. @@ -105,34 +117,80 @@ templates: original_task: true name: random_choice_A_B_2 reference: '' - 58d68674-6069-4954-8433-fd62c3569420: !Template + 2d7eb2d3-780b-4159-aea2-2126a0e4eb9e: !Template + answer_choices: null + id: 2d7eb2d3-780b-4159-aea2-2126a0e4eb9e + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2_2 + reference: '' + 33816ce2-a5e8-4c69-85ab-08ba1bcaa743: !Template answer_choices: Yes ||| No - id: 58d68674-6069-4954-8433-fd62c3569420 - jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] - }} + id: 33816ce2-a5e8-4c69-85ab-08ba1bcaa743 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata - choices_in_prompt: false + choices_in_prompt: true metrics: - Accuracy original_task: false - name: Yes/No binary judgment for bad sentences + name: Yes/No binary judgment for good sentences reference: '' - 78a22835-3107-4dda-a6eb-a3d347714fdc: !Template - answer_choices: B ||| A - id: 78a22835-3107-4dda-a6eb-a3d347714fdc - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] - }}' + 4239e2b0-a994-409f-a01a-505765885096: !Template + answer_choices: Yes ||| No + id: 4239e2b0-a994-409f-a01a-505765885096 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'B/A choice, 2 (bad-good, answer: B)' + original_task: false + name: Yes/No binary judgment for bad sentences reference: '' - 8a96d69b-d9f3-42cb-bb17-0b4020f47df0: !Template + 45b30932-072c-48b4-9134-93ff1ca0656f: !Template answer_choices: null - id: 8a96d69b-d9f3-42cb-bb17-0b4020f47df0 + id: 45b30932-072c-48b4-9134-93ff1ca0656f jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer A or B. @@ -169,36 +227,67 @@ templates: original_task: true name: random_choice_A_B reference: '' - 9d4ae77b-6681-4d61-be95-cd4d7318169c: !Template - answer_choices: null - id: 9d4ae77b-6681-4d61-be95-cd4d7318169c + 8b4608d9-5243-4696-b2bb-c5ee0c130aa7: !Template + answer_choices: Yes ||| No + id: 8b4608d9-5243-4696-b2bb-c5ee0c130aa7 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences (choice order reversed) + reference: '' + a397a1a9-9c20-429a-9c9f-bd423e004332: !Template + answer_choices: Yes ||| No + id: a397a1a9-9c20-429a-9c9f-bd423e004332 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences (choice order reversed) + reference: '' + d463b428-ebd8-4cb6-a70e-d4316a829a0e: !Template + answer_choices: A ||| B + id: d463b428-ebd8-4cb6-a70e-d4316a829a0e jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} - {% if shuffled_order == 0 %} - - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -206,30 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 - reference: '' - ae170596-102a-4eb7-8d13-e68acb79c4f6: !Template - answer_choices: A ||| B - id: ae170596-102a-4eb7-8d13-e68acb79c4f6 - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'A/B choice 2 (bad-good, answer: B)' - reference: '' - b83d9f01-6f15-4c93-8562-6a5e20249a05: !Template - answer_choices: Yes ||| No - id: b83d9f01-6f15-4c93-8562-6a5e20249a05 - jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] - }} - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences + name: A/B choice randomized reference: '' diff --git a/promptsource/templates/blimp/principle_A_domain_3/templates.yaml b/promptsource/templates/blimp/principle_A_domain_3/templates.yaml index c62e9f91f..d903b1220 100644 --- a/promptsource/templates/blimp/principle_A_domain_3/templates.yaml +++ b/promptsource/templates/blimp/principle_A_domain_3/templates.yaml @@ -1,37 +1,94 @@ dataset: blimp subset: principle_A_domain_3 templates: - 0df0c10f-d880-4201-85ec-de46124137e0: !Template + 017c8bef-e473-4c98-8e72-e27f84c0653f: !Template + answer_choices: Yes ||| No + id: 017c8bef-e473-4c98-8e72-e27f84c0653f + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences (choice order reversed) + reference: '' + 38bbc0c5-a9a3-4d8b-938e-bd66878d780a: !Template + answer_choices: Yes ||| No + id: 38bbc0c5-a9a3-4d8b-938e-bd66878d780a + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' + 632cb9c1-23a4-4744-b3c4-c606e6ee36e1: !Template answer_choices: A ||| B - id: 0df0c10f-d880-4201-85ec-de46124137e0 - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] - }}' + id: 632cb9c1-23a4-4744-b3c4-c606e6ee36e1 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'A/B choice 1 (good-bad, answer: A)' + name: A/B choice randomized (choice order reversed) reference: '' - 187b10eb-3242-49d6-b23c-474356347392: !Template + 6be19da9-f28c-4620-9215-324a42f9d6ba: !Template answer_choices: null - id: 187b10eb-3242-49d6-b23c-474356347392 + id: 6be19da9-f28c-4620-9215-324a42f9d6ba jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -39,11 +96,11 @@ templates: {% if shuffled_order == 0 %} - A + 1 {% else %} - B + 2 {% endif %}' metadata: !TemplateMetadata @@ -51,11 +108,65 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: random_choice_1_2_2 + reference: '' + 8da8d714-93f9-4f16-8115-270b98fd7c9a: !Template + answer_choices: Yes ||| No + id: 8da8d714-93f9-4f16-8115-270b98fd7c9a + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences (choice order reversed) + reference: '' + 9bb59a65-bf8a-4ec1-90d9-44956b266217: !Template + answer_choices: A ||| B + id: 9bb59a65-bf8a-4ec1-90d9-44956b266217 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: A/B choice randomized reference: '' - 24daac32-406a-470b-863b-a96326ae120d: !Template + a68a23b5-fbc8-4fbf-9b55-ba6e7186e893: !Template answer_choices: null - id: 24daac32-406a-470b-863b-a96326ae120d + id: a68a23b5-fbc8-4fbf-9b55-ba6e7186e893 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer A or B. @@ -92,12 +203,12 @@ templates: original_task: true name: random_choice_A_B_2 reference: '' - 4d88ab78-315f-4968-a4b2-894b2a551b65: !Template + bc28de55-b1c6-4efc-bbde-5b7893e145e5: !Template answer_choices: null - id: 4d88ab78-315f-4968-a4b2-894b2a551b65 + id: bc28de55-b1c6-4efc-bbde-5b7893e145e5 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -129,77 +240,26 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 - reference: '' - a764efa4-32fa-4221-b60a-2b35a0e319a0: !Template - answer_choices: B ||| A - id: a764efa4-32fa-4221-b60a-2b35a0e319a0 - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'B/A choice, 2 (bad-good, answer: B)' - reference: '' - b6481098-3a97-488c-bb64-67a3a80c9a93: !Template - answer_choices: A ||| B - id: b6481098-3a97-488c-bb64-67a3a80c9a93 - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'A/B choice 2 (bad-good, answer: B)' - reference: '' - cbac2443-7ed6-4ece-834b-eb11a5e329fc: !Template - answer_choices: B ||| A - id: cbac2443-7ed6-4ece-834b-eb11a5e329fc - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'B/A choice 1 (good-bad, answer: A)' - reference: '' - ce9299b1-7dd3-421e-8e95-fb132dbf80c4: !Template - answer_choices: Yes ||| No - id: ce9299b1-7dd3-421e-8e95-fb132dbf80c4 - jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] - }} - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences + name: random_choice_1_2 reference: '' - d9fb33e0-fc95-4f82-8777-460cdcc9e353: !Template + c02161bc-a48d-4b12-8766-3f000b02ed1b: !Template answer_choices: null - id: d9fb33e0-fc95-4f82-8777-460cdcc9e353 + id: c02161bc-a48d-4b12-8766-3f000b02ed1b jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -207,11 +267,11 @@ templates: {% if shuffled_order == 0 %} - 1 + A {% else %} - 2 + B {% endif %}' metadata: !TemplateMetadata @@ -219,17 +279,21 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: random_choice_A_B reference: '' - efa9b032-4ab1-4772-97d9-38e386b9b499: !Template + f8c7071b-0938-48a3-aec0-7b33cb947bf1: !Template answer_choices: Yes ||| No - id: efa9b032-4ab1-4772-97d9-38e386b9b499 - jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] - }} + id: f8c7071b-0938-48a3-aec0-7b33cb947bf1 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata - choices_in_prompt: false + choices_in_prompt: true metrics: - Accuracy original_task: false - name: Yes/No binary judgment for good sentences + name: Yes/No binary judgment for bad sentences reference: '' diff --git a/promptsource/templates/blimp/principle_A_reconstruction/templates.yaml b/promptsource/templates/blimp/principle_A_reconstruction/templates.yaml index 63cec742d..91f5d778f 100644 --- a/promptsource/templates/blimp/principle_A_reconstruction/templates.yaml +++ b/promptsource/templates/blimp/principle_A_reconstruction/templates.yaml @@ -1,75 +1,128 @@ dataset: blimp subset: principle_A_reconstruction templates: - 576c52f7-ac80-4917-8bce-eb280a64d11e: !Template + 0db5847d-aff9-4f57-8acc-4648c6d3f7ba: !Template answer_choices: A ||| B - id: 576c52f7-ac80-4917-8bce-eb280a64d11e - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] - }}' + id: 0db5847d-aff9-4f57-8acc-4648c6d3f7ba + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'A/B choice 1 (good-bad, answer: A)' + name: A/B choice randomized reference: '' - 5e4a07d1-7b3c-4c0f-b4b0-b06dd8917521: !Template - answer_choices: B ||| A - id: 5e4a07d1-7b3c-4c0f-b4b0-b06dd8917521 - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] - }}' + 34a7dafe-8bcb-49d8-9287-618ece2fd349: !Template + answer_choices: null + id: 34a7dafe-8bcb-49d8-9287-618ece2fd349 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'B/A choice 1 (good-bad, answer: A)' + name: random_choice_A_B_2 reference: '' - 5e9d0209-9b95-44ac-aeea-030796c5a1b4: !Template - answer_choices: A ||| B - id: 5e9d0209-9b95-44ac-aeea-030796c5a1b4 - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] - }}' + 34e77fcf-d6dd-47a2-8a9c-5e0802f8f30f: !Template + answer_choices: Yes ||| No + id: 34e77fcf-d6dd-47a2-8a9c-5e0802f8f30f + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'A/B choice 2 (bad-good, answer: B)' + original_task: false + name: Yes/No binary judgment for good sentences reference: '' - 620f2ec7-b155-4c44-8aaf-10fdbfb21b6d: !Template - answer_choices: null - id: 620f2ec7-b155-4c44-8aaf-10fdbfb21b6d + 39205871-125d-40a7-8efa-4319539b9084: !Template + answer_choices: A ||| B + id: 39205871-125d-40a7-8efa-4319539b9084 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -77,14 +130,30 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: A/B choice randomized (choice order reversed) reference: '' - 84d9a2b5-77bc-4280-936f-e909827fdb3b: !Template + 5449f663-83f0-4031-9a00-51a08ef203a8: !Template + answer_choices: Yes ||| No + id: 5449f663-83f0-4031-9a00-51a08ef203a8 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences (choice order reversed) + reference: '' + 91aa5feb-37e6-443f-90f1-b3ca15a79ea5: !Template answer_choices: null - id: 84d9a2b5-77bc-4280-936f-e909827fdb3b + id: 91aa5feb-37e6-443f-90f1-b3ca15a79ea5 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -116,14 +185,46 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: random_choice_A_B reference: '' - 9d6c0bce-f8fd-48b7-b0d6-055f123f6c1c: !Template + 941e5f65-136e-4dfb-be5f-708a90a14a71: !Template + answer_choices: Yes ||| No + id: 941e5f65-136e-4dfb-be5f-708a90a14a71 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' + 95a5c144-7ffa-4a81-9a63-17a1ea8c4e86: !Template + answer_choices: Yes ||| No + id: 95a5c144-7ffa-4a81-9a63-17a1ea8c4e86 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences (choice order reversed) + reference: '' + aa72a26f-7e08-460b-96be-f2396dda07e2: !Template answer_choices: null - id: 9d6c0bce-f8fd-48b7-b0d6-055f123f6c1c + id: aa72a26f-7e08-460b-96be-f2396dda07e2 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -155,26 +256,26 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: random_choice_1_2_2 reference: '' - 9fc16a9e-d9bb-4faa-8f93-e18ed334080f: !Template + cdb05cd3-5728-409d-aa66-1dc310be2db7: !Template answer_choices: null - id: 9fc16a9e-d9bb-4faa-8f93-e18ed334080f + id: cdb05cd3-5728-409d-aa66-1dc310be2db7 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -182,11 +283,11 @@ templates: {% if shuffled_order == 0 %} - A + 1 {% else %} - B + 2 {% endif %}' metadata: !TemplateMetadata @@ -194,42 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B - reference: '' - a1ac5948-bf3d-481c-9a77-6829bcad634c: !Template - answer_choices: B ||| A - id: a1ac5948-bf3d-481c-9a77-6829bcad634c - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'B/A choice, 2 (bad-good, answer: B)' - reference: '' - a3f11f84-0b45-41c7-a661-e08cad572723: !Template - answer_choices: Yes ||| No - id: a3f11f84-0b45-41c7-a661-e08cad572723 - jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] - }} - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences - reference: '' - b1b994ed-a369-438e-9ce4-69314c4ea4d5: !Template - answer_choices: Yes ||| No - id: b1b994ed-a369-438e-9ce4-69314c4ea4d5 - jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] - }} - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences + name: random_choice_1_2 reference: '' diff --git a/promptsource/templates/blimp/regular_plural_subject_verb_agreement_1/templates.yaml b/promptsource/templates/blimp/regular_plural_subject_verb_agreement_1/templates.yaml index 8443cbd01..856845f23 100644 --- a/promptsource/templates/blimp/regular_plural_subject_verb_agreement_1/templates.yaml +++ b/promptsource/templates/blimp/regular_plural_subject_verb_agreement_1/templates.yaml @@ -1,34 +1,86 @@ dataset: blimp subset: regular_plural_subject_verb_agreement_1 templates: - 09aa27be-636f-4d87-b914-035061a4793d: !Template - answer_choices: Yes ||| No - id: 09aa27be-636f-4d87-b914-035061a4793d - jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] - }} + 014a48b2-38c4-4078-8c56-1b8f6891c3c5: !Template + answer_choices: A ||| B + id: 014a48b2-38c4-4078-8c56-1b8f6891c3c5 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata - choices_in_prompt: false + choices_in_prompt: true metrics: - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences + original_task: true + name: A/B choice randomized reference: '' - 349f6fe8-4e6e-40f3-bdba-372ce0ba97bb: !Template - answer_choices: A ||| B - id: 349f6fe8-4e6e-40f3-bdba-372ce0ba97bb - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] - }}' + 2872aff2-8915-4372-b851-68c4080c2b75: !Template + answer_choices: null + id: 2872aff2-8915-4372-b851-68c4080c2b75 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'A/B choice 2 (bad-good, answer: B)' + name: random_choice_A_B_2 reference: '' - 4080b351-4025-4763-9c70-08c6e7afb633: !Template + 29af5cb8-8c9f-4f12-a826-54efec4a6436: !Template answer_choices: null - id: 4080b351-4025-4763-9c70-08c6e7afb633 + id: 29af5cb8-8c9f-4f12-a826-54efec4a6436 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -65,22 +117,41 @@ templates: original_task: true name: random_choice_1_2_2 reference: '' - 443abdb9-dede-4bf2-aada-6cc4f81669f3: !Template - answer_choices: A ||| B - id: 443abdb9-dede-4bf2-aada-6cc4f81669f3 - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] - }}' + 4f776d97-9192-4504-a702-b2ddbe0c0241: !Template + answer_choices: Yes ||| No + id: 4f776d97-9192-4504-a702-b2ddbe0c0241 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'A/B choice 1 (good-bad, answer: A)' + original_task: false + name: Yes/No binary judgment for bad sentences (choice order reversed) + reference: '' + 662c389e-8f04-405d-a833-2e4a540de426: !Template + answer_choices: Yes ||| No + id: 662c389e-8f04-405d-a833-2e4a540de426 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences reference: '' - 65315901-6f9d-42e9-9e35-48ca2326531e: !Template + 6ae71758-f2c6-436f-a33a-8c7169022c0a: !Template answer_choices: null - id: 65315901-6f9d-42e9-9e35-48ca2326531e + id: 6ae71758-f2c6-436f-a33a-8c7169022c0a jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer 1 or 2. @@ -117,48 +188,67 @@ templates: original_task: true name: random_choice_1_2 reference: '' - 7f455113-5d3e-405e-88da-0628671e1311: !Template + 91c0af57-fa62-427a-8835-ed4bff56e782: !Template answer_choices: Yes ||| No - id: 7f455113-5d3e-405e-88da-0628671e1311 - jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] - }} + id: 91c0af57-fa62-427a-8835-ed4bff56e782 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata - choices_in_prompt: false + choices_in_prompt: true metrics: - Accuracy original_task: false - name: Yes/No binary judgment for good sentences + name: Yes/No binary judgment for good sentences (choice order reversed) reference: '' - df32b1f8-cb95-4bc5-a218-91127a52e809: !Template - answer_choices: null - id: df32b1f8-cb95-4bc5-a218-91127a52e809 + d23cc85c-2a08-4323-ac81-475912fe4373: !Template + answer_choices: Yes ||| No + id: d23cc85c-2a08-4323-ac81-475912fe4373 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' + e7015947-5ae2-47d5-a2a6-9a9708323fd3: !Template + answer_choices: A ||| B + id: e7015947-5ae2-47d5-a2a6-9a9708323fd3 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -166,27 +256,14 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B - reference: '' - e5a5ae78-5008-4afa-a306-fbea636a8983: !Template - answer_choices: B ||| A - id: e5a5ae78-5008-4afa-a306-fbea636a8983 - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'B/A choice 1 (good-bad, answer: A)' + name: A/B choice randomized (choice order reversed) reference: '' - f7c5743d-7dc3-4d30-b148-50698897a2af: !Template + f3f6cc90-6521-4154-bfc7-f86a417f91f7: !Template answer_choices: null - id: f7c5743d-7dc3-4d30-b148-50698897a2af + id: f3f6cc90-6521-4154-bfc7-f86a417f91f7 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -218,18 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 - reference: '' - fd31f8b9-923f-4c77-b595-1cd17f52bbb5: !Template - answer_choices: B ||| A - id: fd31f8b9-923f-4c77-b595-1cd17f52bbb5 - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'B/A choice, 2 (bad-good, answer: B)' + name: random_choice_A_B reference: '' diff --git a/promptsource/templates/blimp/regular_plural_subject_verb_agreement_2/templates.yaml b/promptsource/templates/blimp/regular_plural_subject_verb_agreement_2/templates.yaml index 2b6b5c122..3f83f9c11 100644 --- a/promptsource/templates/blimp/regular_plural_subject_verb_agreement_2/templates.yaml +++ b/promptsource/templates/blimp/regular_plural_subject_verb_agreement_2/templates.yaml @@ -1,47 +1,79 @@ dataset: blimp subset: regular_plural_subject_verb_agreement_2 templates: - 0061188f-179e-4654-96be-13267fe36cc2: !Template - answer_choices: Yes ||| No - id: 0061188f-179e-4654-96be-13267fe36cc2 - jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] - }} + 1b5c6c7d-ad79-4d6d-ae75-c19bff02b5a1: !Template + answer_choices: A ||| B + id: 1b5c6c7d-ad79-4d6d-ae75-c19bff02b5a1 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata - choices_in_prompt: false + choices_in_prompt: true metrics: - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences + original_task: true + name: A/B choice randomized reference: '' - 02710ea9-c648-467f-adf8-828aae786848: !Template - answer_choices: B ||| A - id: 02710ea9-c648-467f-adf8-828aae786848 - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] - }}' + 37b6791e-fcd6-4d2b-9b50-83ff3691e56f: !Template + answer_choices: Yes ||| No + id: 37b6791e-fcd6-4d2b-9b50-83ff3691e56f + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'B/A choice 1 (good-bad, answer: A)' + original_task: false + name: Yes/No binary judgment for good sentences reference: '' - 0b908939-2173-4c96-aa68-427092225e12: !Template - answer_choices: A ||| B - id: 0b908939-2173-4c96-aa68-427092225e12 - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] - }}' + 3af4390c-3587-47a9-a67c-f52fca276a89: !Template + answer_choices: Yes ||| No + id: 3af4390c-3587-47a9-a67c-f52fca276a89 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'A/B choice 2 (bad-good, answer: B)' + original_task: false + name: Yes/No binary judgment for bad sentences reference: '' - 50549b1a-008d-474e-9129-f8cd9a0ed949: !Template + 4a05dfc5-2cc4-421b-9e10-64928c8d83c2: !Template answer_choices: null - id: 50549b1a-008d-474e-9129-f8cd9a0ed949 + id: 4a05dfc5-2cc4-421b-9e10-64928c8d83c2 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer A or B. @@ -78,24 +110,24 @@ templates: original_task: true name: random_choice_A_B_2 reference: '' - 61e70ece-6904-4a2d-b8c5-f3ac8b2df65a: !Template + 50d8598a-f818-41a8-ba21-550205105d2b: !Template answer_choices: null - id: 61e70ece-6904-4a2d-b8c5-f3ac8b2df65a + id: 50d8598a-f818-41a8-ba21-550205105d2b jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -103,11 +135,11 @@ templates: {% if shuffled_order == 0 %} - A + 1 {% else %} - B + 2 {% endif %}' metadata: !TemplateMetadata @@ -115,51 +147,42 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B - reference: '' - 8091fa97-efb9-4bfd-8d67-4395e694dec2: !Template - answer_choices: B ||| A - id: 8091fa97-efb9-4bfd-8d67-4395e694dec2 - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'B/A choice, 2 (bad-good, answer: B)' + name: random_choice_1_2 reference: '' - 904d82e6-4d0f-4edb-9632-a2cc7ad6feea: !Template + 64b4bc3c-af5e-4b54-90ec-34a81ecd249b: !Template answer_choices: Yes ||| No - id: 904d82e6-4d0f-4edb-9632-a2cc7ad6feea - jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] - }} + id: 64b4bc3c-af5e-4b54-90ec-34a81ecd249b + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata - choices_in_prompt: false + choices_in_prompt: true metrics: - Accuracy original_task: false - name: Yes/No binary judgment for bad sentences + name: Yes/No binary judgment for bad sentences (choice order reversed) reference: '' - 9575a77e-f5d4-4963-931a-1629eb831b3e: !Template + 759b24f0-0f40-4123-ac11-f21b56a906df: !Template answer_choices: null - id: 9575a77e-f5d4-4963-931a-1629eb831b3e + id: 759b24f0-0f40-4123-ac11-f21b56a906df jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -167,11 +190,11 @@ templates: {% if shuffled_order == 0 %} - 1 + A {% else %} - 2 + B {% endif %}' metadata: !TemplateMetadata @@ -179,24 +202,65 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: random_choice_A_B reference: '' - a5b8bdd4-fe99-423e-b5bc-beaf48ab6f40: !Template + 9dfdce03-a939-414a-a465-617adc0f6460: !Template answer_choices: A ||| B - id: a5b8bdd4-fe99-423e-b5bc-beaf48ab6f40 - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] - }}' + id: 9dfdce03-a939-414a-a465-617adc0f6460 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'A/B choice 1 (good-bad, answer: A)' + name: A/B choice randomized (choice order reversed) + reference: '' + d53410a3-0546-4af1-9df7-113a5a69925f: !Template + answer_choices: Yes ||| No + id: d53410a3-0546-4af1-9df7-113a5a69925f + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences (choice order reversed) reference: '' - db57f37c-26f5-43d4-889b-39ba6fa18c7a: !Template + dd7d0254-5f4f-4eee-92ff-4b860eb419d1: !Template answer_choices: null - id: db57f37c-26f5-43d4-889b-39ba6fa18c7a + id: dd7d0254-5f4f-4eee-92ff-4b860eb419d1 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. diff --git a/promptsource/templates/blimp/sentential_negation_npi_licensor_present/templates.yaml b/promptsource/templates/blimp/sentential_negation_npi_licensor_present/templates.yaml index 4b7d517ce..a055b776c 100644 --- a/promptsource/templates/blimp/sentential_negation_npi_licensor_present/templates.yaml +++ b/promptsource/templates/blimp/sentential_negation_npi_licensor_present/templates.yaml @@ -1,34 +1,79 @@ dataset: blimp subset: sentential_negation_npi_licensor_present templates: - 021b598b-9462-4cc4-a845-284e4bdac7c2: !Template + 1566b195-7965-4789-b33d-2d1d90355e49: !Template + answer_choices: A ||| B + id: 1566b195-7965-4789-b33d-2d1d90355e49 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: A/B choice randomized + reference: '' + 1b7e5684-aeb4-4d8d-9d53-0cef4bb05eaf: !Template answer_choices: Yes ||| No - id: 021b598b-9462-4cc4-a845-284e4bdac7c2 - jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] - }} + id: 1b7e5684-aeb4-4d8d-9d53-0cef4bb05eaf + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata - choices_in_prompt: false + choices_in_prompt: true metrics: - Accuracy original_task: false - name: Yes/No binary judgment for bad sentences + name: Yes/No binary judgment for good sentences reference: '' - 0f887c2a-2018-4716-a317-1e79f6c610cf: !Template - answer_choices: B ||| A - id: 0f887c2a-2018-4716-a317-1e79f6c610cf - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] - }}' + 1ff32cd6-cb3a-49a3-a87d-17813a312db1: !Template + answer_choices: Yes ||| No + id: 1ff32cd6-cb3a-49a3-a87d-17813a312db1 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'B/A choice, 2 (bad-good, answer: B)' + original_task: false + name: Yes/No binary judgment for good sentences (choice order reversed) reference: '' - 28e1631f-5dd1-4018-ba73-f24dc826cb6b: !Template + 265a33b1-d0a8-488c-b0a5-871adbd49bbf: !Template answer_choices: null - id: 28e1631f-5dd1-4018-ba73-f24dc826cb6b + id: 265a33b1-d0a8-488c-b0a5-871adbd49bbf jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer A or B. @@ -65,22 +110,9 @@ templates: original_task: true name: random_choice_A_B reference: '' - 596c3ec0-7c0a-45d3-a036-520f6fabbf4f: !Template - answer_choices: A ||| B - id: 596c3ec0-7c0a-45d3-a036-520f6fabbf4f - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'A/B choice 1 (good-bad, answer: A)' - reference: '' - 61b38dd8-a0c9-4661-a8f4-7168441b73cc: !Template + 379fc292-eb52-442a-b900-8b8466d50199: !Template answer_choices: null - id: 61b38dd8-a0c9-4661-a8f4-7168441b73cc + id: 379fc292-eb52-442a-b900-8b8466d50199 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer 1 or 2. @@ -117,36 +149,51 @@ templates: original_task: true name: random_choice_1_2 reference: '' - 683db107-36f5-417a-9e37-d3c3250a24d8: !Template - answer_choices: null - id: 683db107-36f5-417a-9e37-d3c3250a24d8 + 6913d325-20d1-4712-8a9e-556ede3ddb7b: !Template + answer_choices: Yes ||| No + id: 6913d325-20d1-4712-8a9e-556ede3ddb7b + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' + 8ed2111e-04fe-4d9d-82e0-9a0f0d10c14f: !Template + answer_choices: A ||| B + id: 8ed2111e-04fe-4d9d-82e0-9a0f0d10c14f jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} - {% if shuffled_order == 0 %} - - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -154,49 +201,66 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 - reference: '' - 83ea33f1-6154-4d29-8829-2a2a193943e0: !Template - answer_choices: A ||| B - id: 83ea33f1-6154-4d29-8829-2a2a193943e0 - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'A/B choice 2 (bad-good, answer: B)' + name: A/B choice randomized (choice order reversed) reference: '' - 8864c1f6-e301-46aa-9a17-6157770e6576: !Template + 9f30c4ec-ee1c-4b39-bc13-8aba03d1f937: !Template answer_choices: Yes ||| No - id: 8864c1f6-e301-46aa-9a17-6157770e6576 - jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] - }} + id: 9f30c4ec-ee1c-4b39-bc13-8aba03d1f937 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata - choices_in_prompt: false + choices_in_prompt: true metrics: - Accuracy original_task: false - name: Yes/No binary judgment for good sentences + name: Yes/No binary judgment for bad sentences (choice order reversed) reference: '' - 8af75f75-f09d-4e39-842d-22abf0b5b505: !Template - answer_choices: B ||| A - id: 8af75f75-f09d-4e39-842d-22abf0b5b505 - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] - }}' + b2dbbf76-a5b5-4610-b386-86656690cf3f: !Template + answer_choices: null + id: b2dbbf76-a5b5-4610-b386-86656690cf3f + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'B/A choice 1 (good-bad, answer: A)' + name: random_choice_A_B_2 reference: '' - e9c84f3a-f149-455d-a66e-e33a6196f7f0: !Template + dab171c5-f782-4936-b33c-aefc9b34c6a3: !Template answer_choices: null - id: e9c84f3a-f149-455d-a66e-e33a6196f7f0 + id: dab171c5-f782-4936-b33c-aefc9b34c6a3 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. diff --git a/promptsource/templates/blimp/sentential_negation_npi_scope/templates.yaml b/promptsource/templates/blimp/sentential_negation_npi_scope/templates.yaml index 2572f18ca..8b55678ef 100644 --- a/promptsource/templates/blimp/sentential_negation_npi_scope/templates.yaml +++ b/promptsource/templates/blimp/sentential_negation_npi_scope/templates.yaml @@ -1,48 +1,51 @@ dataset: blimp subset: sentential_negation_npi_scope templates: - 0892df76-cbe9-44d1-9025-003a502c1d64: !Template + 0f7b0f57-f067-4bc2-b028-408443f36ffa: !Template answer_choices: Yes ||| No - id: 0892df76-cbe9-44d1-9025-003a502c1d64 - jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] - }} + id: 0f7b0f57-f067-4bc2-b028-408443f36ffa + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata - choices_in_prompt: false + choices_in_prompt: true metrics: - Accuracy original_task: false - name: Yes/No binary judgment for good sentences + name: Yes/No binary judgment for bad sentences (choice order reversed) reference: '' - 0fa3f3cd-7ba5-4ab9-a532-c104ec7fe277: !Template - answer_choices: null - id: 0fa3f3cd-7ba5-4ab9-a532-c104ec7fe277 + 2c466776-03e8-4029-bb49-9b96f4f3032d: !Template + answer_choices: A ||| B + id: 2c466776-03e8-4029-bb49-9b96f4f3032d jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} - {% if shuffled_order == 0 %} - - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -50,38 +53,85 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: A/B choice randomized (choice order reversed) reference: '' - 1b9e3425-945b-40e9-8115-085e793af225: !Template - answer_choices: null - id: 1b9e3425-945b-40e9-8115-085e793af225 + 39e28834-c2e7-4cdb-b0cd-360b64aaae5f: !Template + answer_choices: Yes ||| No + id: 39e28834-c2e7-4cdb-b0cd-360b64aaae5f + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' + 44d0d846-2859-450a-b546-2872a20013e3: !Template + answer_choices: Yes ||| No + id: 44d0d846-2859-450a-b546-2872a20013e3 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' + a88e1e26-b4e4-4f6d-8582-ad085c44bf54: !Template + answer_choices: Yes ||| No + id: a88e1e26-b4e4-4f6d-8582-ad085c44bf54 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences (choice order reversed) + reference: '' + aa761950-8c2b-47c9-9203-c0bc15c64d14: !Template + answer_choices: A ||| B + id: aa761950-8c2b-47c9-9203-c0bc15c64d14 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -89,37 +139,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 - reference: '' - 29a2acf1-61da-4261-a17a-4be7d7bd99b4: !Template - answer_choices: A ||| B - id: 29a2acf1-61da-4261-a17a-4be7d7bd99b4 - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'A/B choice 2 (bad-good, answer: B)' - reference: '' - 366604a0-b413-4bff-9c2b-5ea9692630bb: !Template - answer_choices: A ||| B - id: 366604a0-b413-4bff-9c2b-5ea9692630bb - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'A/B choice 1 (good-bad, answer: A)' + name: A/B choice randomized reference: '' - 82f02663-648a-4279-ab89-483df722c26a: !Template + b7dcb54b-d02a-4c85-9e52-256ba5daaf8c: !Template answer_choices: null - id: 82f02663-648a-4279-ab89-483df722c26a + id: b7dcb54b-d02a-4c85-9e52-256ba5daaf8c jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer A or B. @@ -156,22 +180,48 @@ templates: original_task: true name: random_choice_A_B_2 reference: '' - 86261239-0be6-4b17-8460-5ec85f2d1f4b: !Template - answer_choices: B ||| A - id: 86261239-0be6-4b17-8460-5ec85f2d1f4b - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] - }}' + b8087f0e-30ca-4a7b-ad59-0f143eb3ec15: !Template + answer_choices: null + id: b8087f0e-30ca-4a7b-ad59-0f143eb3ec15 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'B/A choice, 2 (bad-good, answer: B)' + name: random_choice_A_B reference: '' - f187f967-4029-4281-8ad7-08ac22d910f2: !Template + eac257fa-d2f6-4b36-9805-6aa3e68c4ab1: !Template answer_choices: null - id: f187f967-4029-4281-8ad7-08ac22d910f2 + id: eac257fa-d2f6-4b36-9805-6aa3e68c4ab1 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -208,28 +258,42 @@ templates: original_task: true name: random_choice_1_2_2 reference: '' - f2650ead-ff98-4044-8063-1806dff86a80: !Template - answer_choices: B ||| A - id: f2650ead-ff98-4044-8063-1806dff86a80 - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] - }}' + f4785d78-7953-4332-adc9-89d6df976106: !Template + answer_choices: null + id: f4785d78-7953-4332-adc9-89d6df976106 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'B/A choice 1 (good-bad, answer: A)' - reference: '' - f6d72d15-3cbb-4985-9270-462dabeb1cf4: !Template - answer_choices: Yes ||| No - id: f6d72d15-3cbb-4985-9270-462dabeb1cf4 - jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] - }} - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences + name: random_choice_1_2 reference: '' diff --git a/promptsource/templates/blimp/sentential_subject_island/templates.yaml b/promptsource/templates/blimp/sentential_subject_island/templates.yaml index cdb2b266e..cd8cd07c7 100644 --- a/promptsource/templates/blimp/sentential_subject_island/templates.yaml +++ b/promptsource/templates/blimp/sentential_subject_island/templates.yaml @@ -1,9 +1,25 @@ dataset: blimp subset: sentential_subject_island templates: - 1867a802-40e0-47c0-8c04-8bdac04c9018: !Template + 16fdd0d2-8afa-4df4-a06c-c97c8e9f766d: !Template + answer_choices: Yes ||| No + id: 16fdd0d2-8afa-4df4-a06c-c97c8e9f766d + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences (choice order reversed) + reference: '' + 1f0def75-b22b-4ced-bb79-86a3095cfb7a: !Template answer_choices: null - id: 1867a802-40e0-47c0-8c04-8bdac04c9018 + id: 1f0def75-b22b-4ced-bb79-86a3095cfb7a jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer 1 or 2. @@ -40,25 +56,44 @@ templates: original_task: true name: random_choice_1_2 reference: '' - 186d1ffa-3bd7-4ba0-bbf2-0dc5d7062ce2: !Template - answer_choices: B ||| A - id: 186d1ffa-3bd7-4ba0-bbf2-0dc5d7062ce2 - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] - }}' + 29a052e7-6f9a-4ddd-902b-ba2aa7d779b6: !Template + answer_choices: Yes ||| No + id: 29a052e7-6f9a-4ddd-902b-ba2aa7d779b6 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'B/A choice, 2 (bad-good, answer: B)' + original_task: false + name: Yes/No binary judgment for good sentences (choice order reversed) + reference: '' + 34f43d45-61df-43aa-ae8d-43a58ea464fe: !Template + answer_choices: Yes ||| No + id: 34f43d45-61df-43aa-ae8d-43a58ea464fe + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences reference: '' - 2494a3da-635f-4491-adc9-d4b7d470dcdf: !Template + 36bddd3c-678e-4fad-9d1d-e27ff2d072a5: !Template answer_choices: null - id: 2494a3da-635f-4491-adc9-d4b7d470dcdf + id: 36bddd3c-678e-4fad-9d1d-e27ff2d072a5 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -90,38 +125,37 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: random_choice_A_B_2 reference: '' - 28fdc7d7-5d0c-4cb6-8460-4802e0d5004f: !Template - answer_choices: null - id: 28fdc7d7-5d0c-4cb6-8460-4802e0d5004f + 70a1ae91-8cc3-4465-8b91-6b7a26b2b88a: !Template + answer_choices: A ||| B + id: 70a1ae91-8cc3-4465-8b91-6b7a26b2b88a jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -129,26 +163,14 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 - reference: '' - 50fe301e-a33c-4e24-bd8e-745902916843: !Template - answer_choices: Yes ||| No - id: 50fe301e-a33c-4e24-bd8e-745902916843 - jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] - }} - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences + name: A/B choice randomized (choice order reversed) reference: '' - 7f4c3371-ba5a-4e42-a5e7-974b8cf19af5: !Template + 7a98a195-9982-4cc8-9958-6a673b218893: !Template answer_choices: null - id: 7f4c3371-ba5a-4e42-a5e7-974b8cf19af5 + id: 7a98a195-9982-4cc8-9958-6a673b218893 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -180,56 +202,98 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 - reference: '' - 8daf5406-4c2c-46e4-adff-6827391ba7cd: !Template - answer_choices: Yes ||| No - id: 8daf5406-4c2c-46e4-adff-6827391ba7cd - jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] - }} - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences + name: random_choice_A_B reference: '' - 9f9ad12e-1bab-4959-9596-d3478f396787: !Template + 7cd25f9a-0a2d-452b-a7b3-243d232e5ddd: !Template answer_choices: A ||| B - id: 9f9ad12e-1bab-4959-9596-d3478f396787 - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] - }}' + id: 7cd25f9a-0a2d-452b-a7b3-243d232e5ddd + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'A/B choice 2 (bad-good, answer: B)' + name: A/B choice randomized reference: '' - c3109a85-f78d-4dbe-9cca-24a8800fb4cd: !Template - answer_choices: A ||| B - id: c3109a85-f78d-4dbe-9cca-24a8800fb4cd - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] - }}' + 904692c6-2d60-4721-9eb2-c36d91794716: !Template + answer_choices: null + id: 904692c6-2d60-4721-9eb2-c36d91794716 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'A/B choice 1 (good-bad, answer: A)' + name: random_choice_1_2_2 reference: '' - ee405258-203f-47f9-8f85-ca1654bfa1e3: !Template - answer_choices: B ||| A - id: ee405258-203f-47f9-8f85-ca1654bfa1e3 - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] - }}' + fed02fad-6382-4f21-a66a-98ec3a9ef084: !Template + answer_choices: Yes ||| No + id: fed02fad-6382-4f21-a66a-98ec3a9ef084 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'B/A choice 1 (good-bad, answer: A)' + original_task: false + name: Yes/No binary judgment for good sentences reference: '' diff --git a/promptsource/templates/blimp/superlative_quantifiers_1/templates.yaml b/promptsource/templates/blimp/superlative_quantifiers_1/templates.yaml index 451854807..cae680695 100644 --- a/promptsource/templates/blimp/superlative_quantifiers_1/templates.yaml +++ b/promptsource/templates/blimp/superlative_quantifiers_1/templates.yaml @@ -1,37 +1,28 @@ dataset: blimp subset: superlative_quantifiers_1 templates: - 11ccc1f4-2e68-47db-8493-5442f179e952: !Template - answer_choices: B ||| A - id: 11ccc1f4-2e68-47db-8493-5442f179e952 - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'B/A choice, 2 (bad-good, answer: B)' - reference: '' - 134066c0-60b0-408c-be42-6c594d4c6af5: !Template + 06382d63-4b72-483b-ae6c-9861bf24f81c: !Template answer_choices: Yes ||| No - id: 134066c0-60b0-408c-be42-6c594d4c6af5 - jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] - }} + id: 06382d63-4b72-483b-ae6c-9861bf24f81c + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata - choices_in_prompt: false + choices_in_prompt: true metrics: - Accuracy original_task: false - name: Yes/No binary judgment for good sentences + name: Yes/No binary judgment for bad sentences reference: '' - 1e124c50-9ab6-4c07-b514-4d9cad071a5d: !Template + 159b2baa-694b-401b-820f-bba21fce82cf: !Template answer_choices: null - id: 1e124c50-9ab6-4c07-b514-4d9cad071a5d + id: 159b2baa-694b-401b-820f-bba21fce82cf jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -63,38 +54,37 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: random_choice_A_B reference: '' - 1ee846e6-133b-4a31-83c1-de959f7d5491: !Template - answer_choices: null - id: 1ee846e6-133b-4a31-83c1-de959f7d5491 + 1f275906-f740-4f5c-a136-2f7beaa19975: !Template + answer_choices: A ||| B + id: 1f275906-f740-4f5c-a136-2f7beaa19975 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -102,51 +92,58 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: A/B choice randomized (choice order reversed) reference: '' - 493c7693-514b-4d3b-91af-6153ecd59061: !Template - answer_choices: A ||| B - id: 493c7693-514b-4d3b-91af-6153ecd59061 - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] - }}' + 2b7e9a30-fb60-4980-beb2-6ec591dc470d: !Template + answer_choices: Yes ||| No + id: 2b7e9a30-fb60-4980-beb2-6ec591dc470d + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'A/B choice 1 (good-bad, answer: A)' + original_task: false + name: Yes/No binary judgment for good sentences reference: '' - 576b9858-f67f-45b8-8cd1-e333737a1f40: !Template + 60574278-67e2-43b7-802a-b77ada04d832: !Template answer_choices: Yes ||| No - id: 576b9858-f67f-45b8-8cd1-e333737a1f40 - jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] - }} + id: 60574278-67e2-43b7-802a-b77ada04d832 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata - choices_in_prompt: false + choices_in_prompt: true metrics: - Accuracy original_task: false - name: Yes/No binary judgment for bad sentences + name: Yes/No binary judgment for good sentences (choice order reversed) reference: '' - 58ed84ff-a099-47c4-ab63-b57b3e71e726: !Template + 62cc186f-85fb-410a-a980-88d6aca187da: !Template answer_choices: null - id: 58ed84ff-a099-47c4-ab63-b57b3e71e726 + id: 62cc186f-85fb-410a-a980-88d6aca187da jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -154,11 +151,11 @@ templates: {% if shuffled_order == 0 %} - A + 1 {% else %} - B + 2 {% endif %}' metadata: !TemplateMetadata @@ -166,27 +163,52 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: random_choice_1_2 reference: '' - 80f84bbd-2ad3-4d57-bcc6-09d6098e020e: !Template + 6ca59cd4-3888-423e-bc5d-ab8380ed7850: !Template answer_choices: A ||| B - id: 80f84bbd-2ad3-4d57-bcc6-09d6098e020e - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] - }}' + id: 6ca59cd4-3888-423e-bc5d-ab8380ed7850 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'A/B choice 2 (bad-good, answer: B)' + name: A/B choice randomized reference: '' - c7cf5971-9888-4ca2-9400-a2c0f3dd42fd: !Template + 876a0627-e0e9-4b42-8dec-662e97f14850: !Template answer_choices: null - id: c7cf5971-9888-4ca2-9400-a2c0f3dd42fd + id: 876a0627-e0e9-4b42-8dec-662e97f14850 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -218,18 +240,60 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: random_choice_1_2_2 reference: '' - ce043cec-56e6-4fd6-9332-60be62e5a22b: !Template - answer_choices: B ||| A - id: ce043cec-56e6-4fd6-9332-60be62e5a22b - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] - }}' + b248fd0b-a990-4db7-a26c-2325729bbbee: !Template + answer_choices: Yes ||| No + id: b248fd0b-a990-4db7-a26c-2325729bbbee + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences (choice order reversed) + reference: '' + ee010f7d-c4e2-4640-8d00-01e2517a78be: !Template + answer_choices: null + id: ee010f7d-c4e2-4640-8d00-01e2517a78be + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'B/A choice 1 (good-bad, answer: A)' + name: random_choice_A_B_2 reference: '' diff --git a/promptsource/templates/blimp/superlative_quantifiers_2/templates.yaml b/promptsource/templates/blimp/superlative_quantifiers_2/templates.yaml index 29c22e6d0..c5c9982c2 100644 --- a/promptsource/templates/blimp/superlative_quantifiers_2/templates.yaml +++ b/promptsource/templates/blimp/superlative_quantifiers_2/templates.yaml @@ -1,9 +1,9 @@ dataset: blimp subset: superlative_quantifiers_2 templates: - 199def9a-fb9e-4594-8eae-e188ae4beb4a: !Template + 0e6b233c-6dc1-4e40-90e2-45c124dba2a6: !Template answer_choices: null - id: 199def9a-fb9e-4594-8eae-e188ae4beb4a + id: 0e6b233c-6dc1-4e40-90e2-45c124dba2a6 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer A or B. @@ -40,21 +40,9 @@ templates: original_task: true name: random_choice_A_B_2 reference: '' - 5165fa6b-b270-4b27-8394-3a6b9712854f: !Template - answer_choices: Yes ||| No - id: 5165fa6b-b270-4b27-8394-3a6b9712854f - jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] - }} - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences - reference: '' - 64300c9d-758c-4091-8977-e9f7e73c899c: !Template + 216e6b95-edf9-432e-a312-a56ba3a7e338: !Template answer_choices: null - id: 64300c9d-758c-4091-8977-e9f7e73c899c + id: 216e6b95-edf9-432e-a312-a56ba3a7e338 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer 1 or 2. @@ -91,74 +79,128 @@ templates: original_task: true name: random_choice_1_2 reference: '' - 8ecc59b8-a388-47ad-b1e3-bd1eea549b9c: !Template - answer_choices: Yes ||| No - id: 8ecc59b8-a388-47ad-b1e3-bd1eea549b9c - jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] - }} + 26285bf8-75dd-46b5-94ba-1740a1e43143: !Template + answer_choices: null + id: 26285bf8-75dd-46b5-94ba-1740a1e43143 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' metadata: !TemplateMetadata - choices_in_prompt: false + choices_in_prompt: true metrics: - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences + original_task: true + name: random_choice_1_2_2 reference: '' - 9309abba-e242-4512-9cee-13e77c6e4753: !Template - answer_choices: B ||| A - id: 9309abba-e242-4512-9cee-13e77c6e4753 - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] - }}' + 3a280cc5-2ffd-45d3-a64d-a8424540d9fb: !Template + answer_choices: Yes ||| No + id: 3a280cc5-2ffd-45d3-a64d-a8424540d9fb + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'B/A choice, 2 (bad-good, answer: B)' + original_task: false + name: Yes/No binary judgment for bad sentences (choice order reversed) reference: '' - 945bec39-e0d9-415e-9808-ff014239515a: !Template + 541fa01a-73f6-4ddc-8861-50d4fcc949ab: !Template answer_choices: A ||| B - id: 945bec39-e0d9-415e-9808-ff014239515a - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] - }}' + id: 541fa01a-73f6-4ddc-8861-50d4fcc949ab + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'A/B choice 1 (good-bad, answer: A)' + name: A/B choice randomized (choice order reversed) reference: '' - a8505fda-81c6-4f24-9b81-21d37241cdf2: !Template - answer_choices: null - id: a8505fda-81c6-4f24-9b81-21d37241cdf2 + 6c7883a7-519f-4b7b-8646-b8c37f0247d1: !Template + answer_choices: A ||| B + id: 6c7883a7-519f-4b7b-8646-b8c37f0247d1 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} - {% if shuffled_order == 0 %} - - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -166,52 +208,58 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: A/B choice randomized reference: '' - b09e0d35-67e7-42b1-8af1-7328a3ebee15: !Template - answer_choices: A ||| B - id: b09e0d35-67e7-42b1-8af1-7328a3ebee15 - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] - }}' + 70808c31-fc16-4343-a107-bed61889e83f: !Template + answer_choices: Yes ||| No + id: 70808c31-fc16-4343-a107-bed61889e83f + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'A/B choice 2 (bad-good, answer: B)' + original_task: false + name: Yes/No binary judgment for bad sentences reference: '' - bd488f67-ebf7-4111-ac11-479db18eff7e: !Template - answer_choices: B ||| A - id: bd488f67-ebf7-4111-ac11-479db18eff7e - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] - }}' + a913f701-6df7-4dcc-9000-735a07e65ef5: !Template + answer_choices: Yes ||| No + id: a913f701-6df7-4dcc-9000-735a07e65ef5 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'B/A choice 1 (good-bad, answer: A)' + original_task: false + name: Yes/No binary judgment for good sentences reference: '' - cb3e52b0-ce14-4a7a-a6e5-ca0bd81de825: !Template + ade2bfa0-df0e-4e13-948c-57e11f33e6b3: !Template answer_choices: null - id: cb3e52b0-ce14-4a7a-a6e5-ca0bd81de825 + id: ade2bfa0-df0e-4e13-948c-57e11f33e6b3 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -219,11 +267,11 @@ templates: {% if shuffled_order == 0 %} - 1 + A {% else %} - 2 + B {% endif %}' metadata: !TemplateMetadata @@ -231,5 +279,21 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: random_choice_A_B + reference: '' + e84c0767-2ec6-4067-9401-b9335e013d39: !Template + answer_choices: Yes ||| No + id: e84c0767-2ec6-4067-9401-b9335e013d39 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences (choice order reversed) reference: '' diff --git a/promptsource/templates/blimp/tough_vs_raising_1/templates.yaml b/promptsource/templates/blimp/tough_vs_raising_1/templates.yaml index 2097f58da..59fc67c85 100644 --- a/promptsource/templates/blimp/tough_vs_raising_1/templates.yaml +++ b/promptsource/templates/blimp/tough_vs_raising_1/templates.yaml @@ -1,62 +1,78 @@ dataset: blimp subset: tough_vs_raising_1 templates: - 3ee110da-2c50-48c1-9b06-c074744d4c76: !Template - answer_choices: Yes ||| No - id: 3ee110da-2c50-48c1-9b06-c074744d4c76 - jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] - }} - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences - reference: '' - 61438af9-e8fd-43b5-8248-518fb7cf22eb: !Template + 1686d3fb-896a-4a5e-9899-9f42d3c7fbc7: !Template answer_choices: A ||| B - id: 61438af9-e8fd-43b5-8248-518fb7cf22eb - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] - }}' + id: 1686d3fb-896a-4a5e-9899-9f42d3c7fbc7 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'A/B choice 1 (good-bad, answer: A)' + name: A/B choice randomized (choice order reversed) reference: '' - 70bb1415-b071-4402-a323-5d605590c5b4: !Template - answer_choices: A ||| B - id: 70bb1415-b071-4402-a323-5d605590c5b4 - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] - }}' + 4abbdd3c-141c-4457-a652-e8f854e1f6b3: !Template + answer_choices: Yes ||| No + id: 4abbdd3c-141c-4457-a652-e8f854e1f6b3 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'A/B choice 2 (bad-good, answer: B)' + original_task: false + name: Yes/No binary judgment for good sentences (choice order reversed) reference: '' - 7e30fe59-f359-434f-8697-e90c8afd892e: !Template + 51185250-89ac-415c-8c2c-aac5322257b1: !Template answer_choices: null - id: 7e30fe59-f359-434f-8697-e90c8afd892e + id: 51185250-89ac-415c-8c2c-aac5322257b1 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -64,11 +80,11 @@ templates: {% if shuffled_order == 0 %} - A + 1 {% else %} - B + 2 {% endif %}' metadata: !TemplateMetadata @@ -76,26 +92,53 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: random_choice_1_2_2 reference: '' - 7fb521b7-26ca-4f44-bce7-e3978960281b: !Template - answer_choices: Yes ||| No - id: 7fb521b7-26ca-4f44-bce7-e3978960281b - jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] - }} + 79bb6de3-5378-4a3d-9d8d-d5e24f63f8a0: !Template + answer_choices: null + id: 79bb6de3-5378-4a3d-9d8d-d5e24f63f8a0 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' metadata: !TemplateMetadata - choices_in_prompt: false + choices_in_prompt: true metrics: - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences + original_task: true + name: random_choice_1_2 reference: '' - 860e2367-ebda-41db-8225-a261f829438e: !Template + 80216836-1950-4484-93a5-09e74b99ec5d: !Template answer_choices: null - id: 860e2367-ebda-41db-8225-a261f829438e + id: 80216836-1950-4484-93a5-09e74b99ec5d jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -127,52 +170,42 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B - reference: '' - 861bf826-2d14-4e1c-946c-28f6927bd7c9: !Template - answer_choices: B ||| A - id: 861bf826-2d14-4e1c-946c-28f6927bd7c9 - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'B/A choice 1 (good-bad, answer: A)' + name: random_choice_A_B_2 reference: '' - 8c774216-21ea-49cd-9256-3bd0f607f9cc: !Template - answer_choices: B ||| A - id: 8c774216-21ea-49cd-9256-3bd0f607f9cc - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] - }}' + 81139d64-3df6-4a97-b823-d3d22e41cc21: !Template + answer_choices: Yes ||| No + id: 81139d64-3df6-4a97-b823-d3d22e41cc21 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'B/A choice, 2 (bad-good, answer: B)' + original_task: false + name: Yes/No binary judgment for bad sentences (choice order reversed) reference: '' - b1562d73-9f9c-42df-ad17-f71b412d86e7: !Template + 95f298f3-b243-4b5c-bb12-070b21165d02: !Template answer_choices: null - id: b1562d73-9f9c-42df-ad17-f71b412d86e7 + id: 95f298f3-b243-4b5c-bb12-070b21165d02 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -180,11 +213,11 @@ templates: {% if shuffled_order == 0 %} - 1 + A {% else %} - 2 + B {% endif %}' metadata: !TemplateMetadata @@ -192,38 +225,37 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: random_choice_A_B reference: '' - f687b5b2-cc13-46a0-884a-d3db1d7aa515: !Template - answer_choices: null - id: f687b5b2-cc13-46a0-884a-d3db1d7aa515 + b367ca33-aabe-43d0-be2e-089f805f4219: !Template + answer_choices: A ||| B + id: b367ca33-aabe-43d0-be2e-089f805f4219 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} - {% if shuffled_order == 0 %} - - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -231,5 +263,37 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: A/B choice randomized + reference: '' + b73e4e71-2b5a-4bae-98a5-f71ac94c44ef: !Template + answer_choices: Yes ||| No + id: b73e4e71-2b5a-4bae-98a5-f71ac94c44ef + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' + e7bf6d42-f4be-4b9d-bee7-cfd05b09a561: !Template + answer_choices: Yes ||| No + id: e7bf6d42-f4be-4b9d-bee7-cfd05b09a561 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences reference: '' diff --git a/promptsource/templates/blimp/tough_vs_raising_2/templates.yaml b/promptsource/templates/blimp/tough_vs_raising_2/templates.yaml index 8a4943201..98acfe535 100644 --- a/promptsource/templates/blimp/tough_vs_raising_2/templates.yaml +++ b/promptsource/templates/blimp/tough_vs_raising_2/templates.yaml @@ -1,9 +1,63 @@ dataset: blimp subset: tough_vs_raising_2 templates: - 125ed492-f4df-4621-908f-7b614eaaf1e1: !Template + 14e649e0-966e-4037-ae98-a619018e0104: !Template + answer_choices: Yes ||| No + id: 14e649e0-966e-4037-ae98-a619018e0104 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences (choice order reversed) + reference: '' + 2a18d5a6-e07e-482e-9e66-b05a6ab05040: !Template + answer_choices: A ||| B + id: 2a18d5a6-e07e-482e-9e66-b05a6ab05040 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: A/B choice randomized + reference: '' + 641ac8f7-8edd-4a87-abd5-8fdff0da9d19: !Template answer_choices: null - id: 125ed492-f4df-4621-908f-7b614eaaf1e1 + id: 641ac8f7-8edd-4a87-abd5-8fdff0da9d19 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer A or B. @@ -40,86 +94,106 @@ templates: original_task: true name: random_choice_A_B_2 reference: '' - 1f82e12a-2abc-4666-b671-b632d0865fb1: !Template + 7a632207-6987-411b-a5d4-57874a2caa82: !Template answer_choices: Yes ||| No - id: 1f82e12a-2abc-4666-b671-b632d0865fb1 - jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] - }} + id: 7a632207-6987-411b-a5d4-57874a2caa82 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata - choices_in_prompt: false + choices_in_prompt: true metrics: - Accuracy original_task: false name: Yes/No binary judgment for bad sentences reference: '' - 437d29bb-252c-49a8-9aa4-85dc5079a195: !Template - answer_choices: B ||| A - id: 437d29bb-252c-49a8-9aa4-85dc5079a195 - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] - }}' + 8031c00b-e407-4bba-9944-f2d1ed5d0839: !Template + answer_choices: Yes ||| No + id: 8031c00b-e407-4bba-9944-f2d1ed5d0839 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'B/A choice, 2 (bad-good, answer: B)' + original_task: false + name: Yes/No binary judgment for good sentences (choice order reversed) reference: '' - 49c1e0e8-e1b3-4f83-b0f1-21aabecfc149: !Template - answer_choices: B ||| A - id: 49c1e0e8-e1b3-4f83-b0f1-21aabecfc149 - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] - }}' + 935bcac4-f021-4532-919b-302db1f4282c: !Template + answer_choices: null + id: 935bcac4-f021-4532-919b-302db1f4282c + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'B/A choice 1 (good-bad, answer: A)' - reference: '' - 5c4cb780-a52a-4e9a-b654-5ea6ec7e05e4: !Template - answer_choices: Yes ||| No - id: 5c4cb780-a52a-4e9a-b654-5ea6ec7e05e4 - jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] - }} - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences + name: random_choice_A_B reference: '' - 65fac625-38ad-4b82-9532-332cb140ee3b: !Template - answer_choices: null - id: 65fac625-38ad-4b82-9532-332cb140ee3b + 99d65ae4-c014-41eb-a3f1-a2b608572668: !Template + answer_choices: A ||| B + id: 99d65ae4-c014-41eb-a3f1-a2b608572668 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -127,11 +201,27 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: A/B choice randomized (choice order reversed) reference: '' - 8e41bb9d-9136-415f-907c-f5a84c043341: !Template + 9fc0d7e6-7b4f-4614-84fc-5cd0c6882513: !Template + answer_choices: Yes ||| No + id: 9fc0d7e6-7b4f-4614-84fc-5cd0c6882513 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' + ad13698d-af51-4145-b12f-e6809075792a: !Template answer_choices: null - id: 8e41bb9d-9136-415f-907c-f5a84c043341 + id: ad13698d-af51-4145-b12f-e6809075792a jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer 1 or 2. @@ -168,24 +258,24 @@ templates: original_task: true name: random_choice_1_2 reference: '' - b97f4ee5-62df-4f8c-9a8a-603ba3d226f7: !Template + f081fe14-16f8-40a3-95fe-cf5b83c81afd: !Template answer_choices: null - id: b97f4ee5-62df-4f8c-9a8a-603ba3d226f7 + id: f081fe14-16f8-40a3-95fe-cf5b83c81afd jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -193,11 +283,11 @@ templates: {% if shuffled_order == 0 %} - A + 1 {% else %} - B + 2 {% endif %}' metadata: !TemplateMetadata @@ -205,31 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B - reference: '' - cc816098-02f5-4071-8b01-ebbc34910f0d: !Template - answer_choices: A ||| B - id: cc816098-02f5-4071-8b01-ebbc34910f0d - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'A/B choice 1 (good-bad, answer: A)' - reference: '' - fdf5e28e-3f15-45a1-8d88-942a4bfac780: !Template - answer_choices: A ||| B - id: fdf5e28e-3f15-45a1-8d88-942a4bfac780 - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'A/B choice 2 (bad-good, answer: B)' + name: random_choice_1_2_2 reference: '' diff --git a/promptsource/templates/blimp/transitive/templates.yaml b/promptsource/templates/blimp/transitive/templates.yaml index 39acd17a1..f5e3ad555 100644 --- a/promptsource/templates/blimp/transitive/templates.yaml +++ b/promptsource/templates/blimp/transitive/templates.yaml @@ -1,73 +1,121 @@ dataset: blimp subset: transitive templates: - 11ba8b9d-b4c8-45b7-8e47-323cd3e5f636: !Template + 0242c638-38a4-44e6-a5b4-b09039c3e9e1: !Template + answer_choices: A ||| B + id: 0242c638-38a4-44e6-a5b4-b09039c3e9e1 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: A/B choice randomized + reference: '' + 259aab32-1ce2-42c3-bd9a-e4f8210fc784: !Template answer_choices: Yes ||| No - id: 11ba8b9d-b4c8-45b7-8e47-323cd3e5f636 - jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] - }} + id: 259aab32-1ce2-42c3-bd9a-e4f8210fc784 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata - choices_in_prompt: false + choices_in_prompt: true metrics: - Accuracy original_task: false name: Yes/No binary judgment for bad sentences reference: '' - 2097f653-2bd7-48f5-9682-57da525636c8: !Template - answer_choices: B ||| A - id: 2097f653-2bd7-48f5-9682-57da525636c8 - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] - }}' + 4f72a2dd-5108-4248-826c-2a86d56f62f7: !Template + answer_choices: Yes ||| No + id: 4f72a2dd-5108-4248-826c-2a86d56f62f7 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'B/A choice 1 (good-bad, answer: A)' + original_task: false + name: Yes/No binary judgment for good sentences (choice order reversed) reference: '' - 2a535f9f-2d23-4875-b458-48fddd66f721: !Template + 4fa2f0ed-854a-41ce-b5c9-776910067e11: !Template answer_choices: Yes ||| No - id: 2a535f9f-2d23-4875-b458-48fddd66f721 - jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] - }} + id: 4fa2f0ed-854a-41ce-b5c9-776910067e11 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata - choices_in_prompt: false + choices_in_prompt: true metrics: - Accuracy original_task: false - name: Yes/No binary judgment for good sentences + name: Yes/No binary judgment for bad sentences (choice order reversed) reference: '' - 2c399b26-3521-4290-83d0-5652e312b5f6: !Template - answer_choices: null - id: 2c399b26-3521-4290-83d0-5652e312b5f6 + 58d8dba9-b98b-4772-b2fb-54837d03a32d: !Template + answer_choices: A ||| B + id: 58d8dba9-b98b-4772-b2fb-54837d03a32d jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} - {% if shuffled_order == 0 %} - - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -75,39 +123,26 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: A/B choice randomized (choice order reversed) reference: '' - 4a0cdb8a-3e9f-4f9f-9184-bfc68f57f41e: !Template - answer_choices: B ||| A - id: 4a0cdb8a-3e9f-4f9f-9184-bfc68f57f41e - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'B/A choice, 2 (bad-good, answer: B)' - reference: '' - 674cdc46-5951-4e84-b44a-701e9d1661a0: !Template + 66bd9d77-b1f0-44fe-a4b1-4a6555ba2897: !Template answer_choices: null - id: 674cdc46-5951-4e84-b44a-701e9d1661a0 + id: 66bd9d77-b1f0-44fe-a4b1-4a6555ba2897 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -115,11 +150,11 @@ templates: {% if shuffled_order == 0 %} - 1 + A {% else %} - 2 + B {% endif %}' metadata: !TemplateMetadata @@ -127,11 +162,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: random_choice_A_B_2 reference: '' - 9ed761fb-b0aa-4322-8b5d-85ad42904150: !Template + 9ee380ca-3ec7-4851-bf7d-d571b014edeb: !Template answer_choices: null - id: 9ed761fb-b0aa-4322-8b5d-85ad42904150 + id: 9ee380ca-3ec7-4851-bf7d-d571b014edeb jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer A or B. @@ -168,37 +203,24 @@ templates: original_task: true name: random_choice_A_B reference: '' - c52ccf34-087a-4ffa-8281-55d839bf6dd0: !Template - answer_choices: A ||| B - id: c52ccf34-087a-4ffa-8281-55d839bf6dd0 - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'A/B choice 2 (bad-good, answer: B)' - reference: '' - d2783acf-666c-4e1f-9597-d23d092e357d: !Template + c1f4ab12-6569-4495-86f4-bbc5552426a4: !Template answer_choices: null - id: d2783acf-666c-4e1f-9597-d23d092e357d + id: c1f4ab12-6569-4495-86f4-bbc5552426a4 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -206,11 +228,11 @@ templates: {% if shuffled_order == 0 %} - A + 1 {% else %} - B + 2 {% endif %}' metadata: !TemplateMetadata @@ -218,18 +240,60 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: random_choice_1_2_2 reference: '' - f39bda9f-f80e-4933-a794-3b907d775b07: !Template - answer_choices: A ||| B - id: f39bda9f-f80e-4933-a794-3b907d775b07 - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] - }}' + ca678159-1031-4ffa-ab54-3953f80e70ba: !Template + answer_choices: Yes ||| No + id: ca678159-1031-4ffa-ab54-3953f80e70ba + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' + e3f3a6c4-34b3-4bcc-995d-53e1724e37b7: !Template + answer_choices: null + id: e3f3a6c4-34b3-4bcc-995d-53e1724e37b7 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'A/B choice 1 (good-bad, answer: A)' + name: random_choice_1_2 reference: '' diff --git a/promptsource/templates/blimp/wh_island/templates.yaml b/promptsource/templates/blimp/wh_island/templates.yaml index bd148ce34..0e64fa92c 100644 --- a/promptsource/templates/blimp/wh_island/templates.yaml +++ b/promptsource/templates/blimp/wh_island/templates.yaml @@ -1,47 +1,64 @@ dataset: blimp subset: wh_island templates: - 0478611e-bb77-4f95-84b8-bd9a23170584: !Template - answer_choices: A ||| B - id: 0478611e-bb77-4f95-84b8-bd9a23170584 - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'A/B choice 2 (bad-good, answer: B)' - reference: '' - 2f809829-d69e-4e3f-bafb-dc14c24d62b1: !Template - answer_choices: A ||| B - id: 2f809829-d69e-4e3f-bafb-dc14c24d62b1 - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] - }}' + 2dca157a-1b3b-4d17-9a66-d2f9ed4cda69: !Template + answer_choices: null + id: 2dca157a-1b3b-4d17-9a66-d2f9ed4cda69 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'A/B choice 1 (good-bad, answer: A)' + name: random_choice_1_2 reference: '' - 3f141165-53f6-4e23-a14d-3c1d0e9f07ff: !Template + 409916e0-3857-4920-9494-05f49404f55a: !Template answer_choices: Yes ||| No - id: 3f141165-53f6-4e23-a14d-3c1d0e9f07ff - jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] - }} + id: 409916e0-3857-4920-9494-05f49404f55a + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata - choices_in_prompt: false + choices_in_prompt: true metrics: - Accuracy original_task: false name: Yes/No binary judgment for good sentences reference: '' - 785b8d5a-1dda-4c2a-b820-36f6c937965f: !Template + 4f6f5f6f-fbe2-425c-8507-b753ca2beccd: !Template answer_choices: null - id: 785b8d5a-1dda-4c2a-b820-36f6c937965f + id: 4f6f5f6f-fbe2-425c-8507-b753ca2beccd jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -78,34 +95,63 @@ templates: original_task: true name: random_choice_1_2_2 reference: '' - 9731dd38-a7b5-4d71-be48-9b550db62440: !Template - answer_choices: B ||| A - id: 9731dd38-a7b5-4d71-be48-9b550db62440 - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] - }}' + 60d5c928-36e2-42ab-87f1-b0335a5ec8a9: !Template + answer_choices: Yes ||| No + id: 60d5c928-36e2-42ab-87f1-b0335a5ec8a9 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'B/A choice 1 (good-bad, answer: A)' + original_task: false + name: Yes/No binary judgment for bad sentences reference: '' - b15bad58-7860-44e3-8158-9c7aa13553da: !Template - answer_choices: Yes ||| No - id: b15bad58-7860-44e3-8158-9c7aa13553da - jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] - }} + 6131e963-2b15-4038-a787-e6fdff231165: !Template + answer_choices: A ||| B + id: 6131e963-2b15-4038-a787-e6fdff231165 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata - choices_in_prompt: false + choices_in_prompt: true metrics: - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences + original_task: true + name: A/B choice randomized (choice order reversed) reference: '' - d78dc0a5-83b2-4e0b-b553-863fe4f06f4c: !Template + 62226dae-1362-499e-b419-e756d455358f: !Template answer_choices: null - id: d78dc0a5-83b2-4e0b-b553-863fe4f06f4c + id: 62226dae-1362-499e-b419-e756d455358f jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer A or B. @@ -142,37 +188,56 @@ templates: original_task: true name: random_choice_A_B reference: '' - e70949d2-9b57-4164-8cab-94a342aecca2: !Template - answer_choices: B ||| A - id: e70949d2-9b57-4164-8cab-94a342aecca2 - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] - }}' + a2e35198-bf39-45d7-ba76-6bcda9454402: !Template + answer_choices: Yes ||| No + id: a2e35198-bf39-45d7-ba76-6bcda9454402 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'B/A choice, 2 (bad-good, answer: B)' + original_task: false + name: Yes/No binary judgment for good sentences (choice order reversed) + reference: '' + c6540e39-22ab-4609-bd50-380ac577edf3: !Template + answer_choices: Yes ||| No + id: c6540e39-22ab-4609-bd50-380ac577edf3 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences (choice order reversed) reference: '' - f1072d69-55eb-435c-8458-8ed626da0d69: !Template + d9f8e6db-52bc-4d96-9822-4e93490771cd: !Template answer_choices: null - id: f1072d69-55eb-435c-8458-8ed626da0d69 + id: d9f8e6db-52bc-4d96-9822-4e93490771cd jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -180,11 +245,11 @@ templates: {% if shuffled_order == 0 %} - 1 + A {% else %} - 2 + B {% endif %}' metadata: !TemplateMetadata @@ -192,38 +257,37 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: random_choice_A_B_2 reference: '' - f1445c4c-fb5d-41a2-b618-be2754c32bba: !Template - answer_choices: null - id: f1445c4c-fb5d-41a2-b618-be2754c32bba + f940fb16-8b1a-4956-b992-e18e3a3a0ada: !Template + answer_choices: A ||| B + id: f940fb16-8b1a-4956-b992-e18e3a3a0ada jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} - {% if shuffled_order == 0 %} - - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -231,5 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: A/B choice randomized reference: '' diff --git a/promptsource/templates/blimp/wh_questions_object_gap/templates.yaml b/promptsource/templates/blimp/wh_questions_object_gap/templates.yaml index 3558630f7..5f65d3804 100644 --- a/promptsource/templates/blimp/wh_questions_object_gap/templates.yaml +++ b/promptsource/templates/blimp/wh_questions_object_gap/templates.yaml @@ -1,50 +1,106 @@ dataset: blimp subset: wh_questions_object_gap templates: - 03396fdd-6d6e-4324-a2e8-19acf2208d1b: !Template - answer_choices: Yes ||| No - id: 03396fdd-6d6e-4324-a2e8-19acf2208d1b - jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] - }} + 0d855a12-4a18-47bd-88d1-700c0e93332a: !Template + answer_choices: null + id: 0d855a12-4a18-47bd-88d1-700c0e93332a + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' metadata: !TemplateMetadata - choices_in_prompt: false + choices_in_prompt: true metrics: - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences + original_task: true + name: random_choice_A_B_2 reference: '' - 1238b84f-7716-4416-a519-7338678392e0: !Template - answer_choices: B ||| A - id: 1238b84f-7716-4416-a519-7338678392e0 - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] - }}' + 10b23d23-7952-47f5-b525-c87c1e015103: !Template + answer_choices: Yes ||| No + id: 10b23d23-7952-47f5-b525-c87c1e015103 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'B/A choice, 2 (bad-good, answer: B)' + original_task: false + name: Yes/No binary judgment for bad sentences reference: '' - 46c0813a-7349-49a2-920e-9e1e95436950: !Template - answer_choices: B ||| A - id: 46c0813a-7349-49a2-920e-9e1e95436950 - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] - }}' + 129d8ec0-b4f0-4ef4-8333-d7d8985a2f5e: !Template + answer_choices: null + id: 129d8ec0-b4f0-4ef4-8333-d7d8985a2f5e + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'B/A choice 1 (good-bad, answer: A)' + name: random_choice_A_B reference: '' - 63f14809-6c03-43be-bcb0-69612e40a79e: !Template + 132135cb-07e3-4615-ab43-16adc300b9c9: !Template answer_choices: null - id: 63f14809-6c03-43be-bcb0-69612e40a79e + id: 132135cb-07e3-4615-ab43-16adc300b9c9 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -76,38 +132,37 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: random_choice_1_2 reference: '' - 66b8a7a0-2b7d-49c6-8f8e-56f7176c4d5d: !Template - answer_choices: null - id: 66b8a7a0-2b7d-49c6-8f8e-56f7176c4d5d + 4d3c0d5c-996c-4b1c-bc6e-649cb1b4d35a: !Template + answer_choices: A ||| B + id: 4d3c0d5c-996c-4b1c-bc6e-649cb1b4d35a jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -115,38 +170,42 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: A/B choice randomized reference: '' - ca560889-5e9c-4b01-a77f-2c21e3720e05: !Template + 54cd4dae-b2f3-4bdc-bcc8-5400a7c4bb6a: !Template answer_choices: Yes ||| No - id: ca560889-5e9c-4b01-a77f-2c21e3720e05 - jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] - }} + id: 54cd4dae-b2f3-4bdc-bcc8-5400a7c4bb6a + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata - choices_in_prompt: false + choices_in_prompt: true metrics: - Accuracy original_task: false - name: Yes/No binary judgment for good sentences + name: Yes/No binary judgment for bad sentences (choice order reversed) reference: '' - cbd4a91a-afd5-4c2b-b77e-3b89c48651bc: !Template + 6d9efb60-2940-494e-8b6e-dc1961a3cd5b: !Template answer_choices: null - id: cbd4a91a-afd5-4c2b-b77e-3b89c48651bc + id: 6d9efb60-2940-494e-8b6e-dc1961a3cd5b jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -154,11 +213,11 @@ templates: {% if shuffled_order == 0 %} - A + 1 {% else %} - B + 2 {% endif %}' metadata: !TemplateMetadata @@ -166,51 +225,53 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: random_choice_1_2_2 reference: '' - d2c46389-8a3a-4df9-aeee-2446d6a8cd90: !Template - answer_choices: A ||| B - id: d2c46389-8a3a-4df9-aeee-2446d6a8cd90 - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] - }}' + cc83c2b5-75c2-45b7-a077-16d7ba336c29: !Template + answer_choices: Yes ||| No + id: cc83c2b5-75c2-45b7-a077-16d7ba336c29 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'A/B choice 2 (bad-good, answer: B)' + original_task: false + name: Yes/No binary judgment for good sentences reference: '' - d864211d-ec04-44ac-a00d-04291ec4298c: !Template - answer_choices: null - id: d864211d-ec04-44ac-a00d-04291ec4298c + e0608d5d-5605-49b2-b9b2-68717a53f2c9: !Template + answer_choices: A ||| B + id: e0608d5d-5605-49b2-b9b2-68717a53f2c9 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -218,18 +279,21 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: A/B choice randomized (choice order reversed) reference: '' - d9f30b1b-df3f-40c6-9268-d5980b971c21: !Template - answer_choices: A ||| B - id: d9f30b1b-df3f-40c6-9268-d5980b971c21 - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] - }}' + f47e01a4-1c90-4d01-b085-c0655cdedc46: !Template + answer_choices: Yes ||| No + id: f47e01a4-1c90-4d01-b085-c0655cdedc46 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'A/B choice 1 (good-bad, answer: A)' + original_task: false + name: Yes/No binary judgment for good sentences (choice order reversed) reference: '' diff --git a/promptsource/templates/blimp/wh_questions_subject_gap/templates.yaml b/promptsource/templates/blimp/wh_questions_subject_gap/templates.yaml index 749c20c41..efdf566a8 100644 --- a/promptsource/templates/blimp/wh_questions_subject_gap/templates.yaml +++ b/promptsource/templates/blimp/wh_questions_subject_gap/templates.yaml @@ -1,22 +1,9 @@ dataset: blimp subset: wh_questions_subject_gap templates: - 1f785875-b311-4dc2-9d21-ec53e25917fa: !Template - answer_choices: B ||| A - id: 1f785875-b311-4dc2-9d21-ec53e25917fa - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'B/A choice 1 (good-bad, answer: A)' - reference: '' - 2e0a281d-6001-48a9-a2f3-8862da48c2a7: !Template + 10166d03-6bdd-4eb9-a621-686310e7dbc3: !Template answer_choices: null - id: 2e0a281d-6001-48a9-a2f3-8862da48c2a7 + id: 10166d03-6bdd-4eb9-a621-686310e7dbc3 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -53,47 +40,9 @@ templates: original_task: true name: random_choice_1_2_2 reference: '' - 4ad52c69-004f-44f5-9549-e8361c05f517: !Template - answer_choices: A ||| B - id: 4ad52c69-004f-44f5-9549-e8361c05f517 - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'A/B choice 1 (good-bad, answer: A)' - reference: '' - 79a7017d-589a-4680-bd49-5ae6584434f5: !Template - answer_choices: Yes ||| No - id: 79a7017d-589a-4680-bd49-5ae6584434f5 - jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] - }} - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences - reference: '' - 79e1d06a-9b7d-41b9-b7bc-5d77476b718b: !Template - answer_choices: A ||| B - id: 79e1d06a-9b7d-41b9-b7bc-5d77476b718b - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'A/B choice 2 (bad-good, answer: B)' - reference: '' - 7d72eb40-e6ca-41f8-8850-cdfd13ad04d5: !Template + 37339aa3-e09b-4c0f-9831-f2aff9bcf11c: !Template answer_choices: null - id: 7d72eb40-e6ca-41f8-8850-cdfd13ad04d5 + id: 37339aa3-e09b-4c0f-9831-f2aff9bcf11c jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer A or B. @@ -130,21 +79,25 @@ templates: original_task: true name: random_choice_A_B reference: '' - 7e7a2b14-e49b-4559-8f97-dc0867d65a1a: !Template + 4d0a99ab-3986-4e8f-8918-b8c4bbf0f4f7: !Template answer_choices: Yes ||| No - id: 7e7a2b14-e49b-4559-8f97-dc0867d65a1a - jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] - }} + id: 4d0a99ab-3986-4e8f-8918-b8c4bbf0f4f7 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata - choices_in_prompt: false + choices_in_prompt: true metrics: - Accuracy original_task: false - name: Yes/No binary judgment for good sentences + name: Yes/No binary judgment for bad sentences (choice order reversed) reference: '' - 868aa4e2-2179-4f75-8ec2-b0499929536c: !Template + 8a0bf278-a9d2-4940-9215-7d3e30e18280: !Template answer_choices: null - id: 868aa4e2-2179-4f75-8ec2-b0499929536c + id: 8a0bf278-a9d2-4940-9215-7d3e30e18280 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer 1 or 2. @@ -181,9 +134,133 @@ templates: original_task: true name: random_choice_1_2 reference: '' - ac68eb62-ac1d-4697-9ded-b7bd76f868d7: !Template + a1c77e55-42a2-4ea7-ab7b-aa967e5b5c76: !Template + answer_choices: A ||| B + id: a1c77e55-42a2-4ea7-ab7b-aa967e5b5c76 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: A/B choice randomized + reference: '' + aa2f941c-3501-42fd-9f68-8a20d23a25b6: !Template + answer_choices: Yes ||| No + id: aa2f941c-3501-42fd-9f68-8a20d23a25b6 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' + b734faa8-895b-44f8-9426-fa45e8664045: !Template + answer_choices: Yes ||| No + id: b734faa8-895b-44f8-9426-fa45e8664045 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' + b982dab8-15b0-4554-94e6-fad6a1d7e490: !Template + answer_choices: A ||| B + id: b982dab8-15b0-4554-94e6-fad6a1d7e490 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: A/B choice randomized (choice order reversed) + reference: '' + c14abdc3-d9a6-4a7e-9e85-698fc20067dd: !Template + answer_choices: Yes ||| No + id: c14abdc3-d9a6-4a7e-9e85-698fc20067dd + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences (choice order reversed) + reference: '' + fd661930-125a-4a9a-97d7-bbab9ba1fd1b: !Template answer_choices: null - id: ac68eb62-ac1d-4697-9ded-b7bd76f868d7 + id: fd661930-125a-4a9a-97d7-bbab9ba1fd1b jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer A or B. @@ -220,16 +297,3 @@ templates: original_task: true name: random_choice_A_B_2 reference: '' - ebff6ddd-90c7-42b5-adbe-d38cb9f6697d: !Template - answer_choices: B ||| A - id: ebff6ddd-90c7-42b5-adbe-d38cb9f6697d - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'B/A choice, 2 (bad-good, answer: B)' - reference: '' diff --git a/promptsource/templates/blimp/wh_questions_subject_gap_long_distance/templates.yaml b/promptsource/templates/blimp/wh_questions_subject_gap_long_distance/templates.yaml index 0111eb941..9e2aaf2ef 100644 --- a/promptsource/templates/blimp/wh_questions_subject_gap_long_distance/templates.yaml +++ b/promptsource/templates/blimp/wh_questions_subject_gap_long_distance/templates.yaml @@ -1,75 +1,116 @@ dataset: blimp subset: wh_questions_subject_gap_long_distance templates: - 013d9cee-cce7-4552-9b4f-464aede20959: !Template - answer_choices: B ||| A - id: 013d9cee-cce7-4552-9b4f-464aede20959 - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] - }}' + 02997aad-8b15-4ce2-96b1-57af313e5b6a: !Template + answer_choices: A ||| B + id: 02997aad-8b15-4ce2-96b1-57af313e5b6a + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'B/A choice, 2 (bad-good, answer: B)' + name: A/B choice randomized (choice order reversed) reference: '' - 2948a7ac-5d31-4f2d-8458-78247861bd2a: !Template - answer_choices: Yes ||| No - id: 2948a7ac-5d31-4f2d-8458-78247861bd2a - jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] - }} - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences - reference: '' - 33e31879-6ab5-412a-aeb2-a77e5f61ef34: !Template - answer_choices: B ||| A - id: 33e31879-6ab5-412a-aeb2-a77e5f61ef34 - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] - }}' + 0ad26bac-7501-448c-b702-0cd0c05781e5: !Template + answer_choices: A ||| B + id: 0ad26bac-7501-448c-b702-0cd0c05781e5 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'B/A choice 1 (good-bad, answer: A)' + name: A/B choice randomized reference: '' - 6ecdd0a7-f8b5-42b8-b011-b4b5221a6082: !Template - answer_choices: A ||| B - id: 6ecdd0a7-f8b5-42b8-b011-b4b5221a6082 - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] - }}' + 17b23ee6-9f95-4fcc-b8d1-65583413879e: !Template + answer_choices: Yes ||| No + id: 17b23ee6-9f95-4fcc-b8d1-65583413879e + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'A/B choice 2 (bad-good, answer: B)' + original_task: false + name: Yes/No binary judgment for bad sentences reference: '' - 9691d735-101f-4753-b9c8-269d8197a443: !Template + 517d74d1-17e3-404c-b7d3-99a91ddb34ff: !Template answer_choices: null - id: 9691d735-101f-4753-b9c8-269d8197a443 + id: 517d74d1-17e3-404c-b7d3-99a91ddb34ff jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -77,11 +118,11 @@ templates: {% if shuffled_order == 0 %} - 1 + A {% else %} - 2 + B {% endif %}' metadata: !TemplateMetadata @@ -89,11 +130,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: random_choice_A_B reference: '' - 9e095ece-29ed-448c-8f1b-2796ce04bced: !Template + 77ba60bb-28cd-4d3e-8a42-53c0bf276a33: !Template answer_choices: null - id: 9e095ece-29ed-448c-8f1b-2796ce04bced + id: 77ba60bb-28cd-4d3e-8a42-53c0bf276a33 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer A or B. @@ -130,49 +171,24 @@ templates: original_task: true name: random_choice_A_B_2 reference: '' - bc82b6e1-c0ae-4b46-ad01-eb7eb9b84745: !Template - answer_choices: Yes ||| No - id: bc82b6e1-c0ae-4b46-ad01-eb7eb9b84745 - jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] - }} - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences - reference: '' - c2543c63-0831-4492-8bad-1833931c63d4: !Template - answer_choices: A ||| B - id: c2543c63-0831-4492-8bad-1833931c63d4 - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'A/B choice 1 (good-bad, answer: A)' - reference: '' - faf4abab-c36f-4750-9ad7-2e7d780c05b3: !Template + 7cd3851b-8a83-408d-b8b5-da92dd0eb653: !Template answer_choices: null - id: faf4abab-c36f-4750-9ad7-2e7d780c05b3 + id: 7cd3851b-8a83-408d-b8b5-da92dd0eb653 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -180,11 +196,11 @@ templates: {% if shuffled_order == 0 %} - A + 1 {% else %} - B + 2 {% endif %}' metadata: !TemplateMetadata @@ -192,11 +208,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: random_choice_1_2 reference: '' - fb6b40ec-bada-4feb-b17e-403ce4d60d9f: !Template + 83ea467e-deda-408e-9d65-5e56c8a88614: !Template answer_choices: null - id: fb6b40ec-bada-4feb-b17e-403ce4d60d9f + id: 83ea467e-deda-408e-9d65-5e56c8a88614 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -233,3 +249,51 @@ templates: original_task: true name: random_choice_1_2_2 reference: '' + a6acbe8c-93ed-43c4-92da-846c2e33b17e: !Template + answer_choices: Yes ||| No + id: a6acbe8c-93ed-43c4-92da-846c2e33b17e + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences + reference: '' + cc00df16-e40a-47f0-b290-b891621b173a: !Template + answer_choices: Yes ||| No + id: cc00df16-e40a-47f0-b290-b891621b173a + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences (choice order reversed) + reference: '' + db0dfb85-8fca-4470-8582-0842152a01b5: !Template + answer_choices: Yes ||| No + id: db0dfb85-8fca-4470-8582-0842152a01b5 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences (choice order reversed) + reference: '' diff --git a/promptsource/templates/blimp/wh_vs_that_no_gap/templates.yaml b/promptsource/templates/blimp/wh_vs_that_no_gap/templates.yaml index 6afcbc645..7f664b9ac 100644 --- a/promptsource/templates/blimp/wh_vs_that_no_gap/templates.yaml +++ b/promptsource/templates/blimp/wh_vs_that_no_gap/templates.yaml @@ -1,25 +1,12 @@ dataset: blimp subset: wh_vs_that_no_gap templates: - 2048ce44-7631-4a28-aa1f-e246af495dd3: !Template - answer_choices: A ||| B - id: 2048ce44-7631-4a28-aa1f-e246af495dd3 - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'A/B choice 2 (bad-good, answer: B)' - reference: '' - 4ba7fc41-f60b-4a17-a4be-7f895de84a5a: !Template + 2caf6cd8-7580-44a0-adb1-eac711d23513: !Template answer_choices: null - id: 4ba7fc41-f60b-4a17-a4be-7f895de84a5a + id: 2caf6cd8-7580-44a0-adb1-eac711d23513 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -51,38 +38,53 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: random_choice_1_2_2 reference: '' - 55d6d95e-1787-4fc7-b536-794afef94c81: !Template - answer_choices: null - id: 55d6d95e-1787-4fc7-b536-794afef94c81 + 6bd58b9b-9d0b-4856-a5c8-92ab4883d700: !Template + answer_choices: Yes ||| No + id: 6bd58b9b-9d0b-4856-a5c8-92ab4883d700 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for good sentences (choice order reversed) + reference: '' + 83f6a959-8a8d-47c7-a78d-e0f10ed8628f: !Template + answer_choices: A ||| B + id: 83f6a959-8a8d-47c7-a78d-e0f10ed8628f jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -90,65 +92,101 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: A/B choice randomized reference: '' - 9cc4732b-7fa3-4bcc-bf18-d19113463db2: !Template - answer_choices: B ||| A - id: 9cc4732b-7fa3-4bcc-bf18-d19113463db2 - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] - }}' + addc8a1c-1c2a-4a76-9a2a-0eb65701ba5a: !Template + answer_choices: Yes ||| No + id: addc8a1c-1c2a-4a76-9a2a-0eb65701ba5a + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'B/A choice 1 (good-bad, answer: A)' + original_task: false + name: Yes/No binary judgment for bad sentences (choice order reversed) reference: '' - a0a33ec9-1c28-4b03-ba0c-3df3ceb64db1: !Template - answer_choices: A ||| B - id: a0a33ec9-1c28-4b03-ba0c-3df3ceb64db1 - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] - }}' + c1086e63-290c-4ccb-a533-22e37825bfcb: !Template + answer_choices: Yes ||| No + id: c1086e63-290c-4ccb-a533-22e37825bfcb + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'A/B choice 1 (good-bad, answer: A)' + original_task: false + name: Yes/No binary judgment for good sentences reference: '' - b0ec8021-bc27-45cb-80cd-54622f8f6eea: !Template - answer_choices: B ||| A - id: b0ec8021-bc27-45cb-80cd-54622f8f6eea - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] - }}' + c8bf7a4d-2415-43c8-88be-6835ee6f9d4e: !Template + answer_choices: null + id: c8bf7a4d-2415-43c8-88be-6835ee6f9d4e + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'B/A choice, 2 (bad-good, answer: B)' + name: random_choice_1_2 reference: '' - c63932ed-7a24-416a-ad09-bcfee969322b: !Template + ce494cf7-6df6-4ec2-b329-5d7d5d8ba2bc: !Template answer_choices: Yes ||| No - id: c63932ed-7a24-416a-ad09-bcfee969322b - jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] - }} + id: ce494cf7-6df6-4ec2-b329-5d7d5d8ba2bc + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata - choices_in_prompt: false + choices_in_prompt: true metrics: - Accuracy original_task: false name: Yes/No binary judgment for bad sentences reference: '' - da3309a0-3693-4cdb-be2b-2b6cb81b41e6: !Template + e529f7d4-ebbf-4240-9616-01b3761b5bcd: !Template answer_choices: null - id: da3309a0-3693-4cdb-be2b-2b6cb81b41e6 + id: e529f7d4-ebbf-4240-9616-01b3761b5bcd jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -180,14 +218,52 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: random_choice_A_B_2 + reference: '' + ef36d150-f6d0-4375-b6bc-6b8339a0a8bc: !Template + answer_choices: A ||| B + id: ef36d150-f6d0-4375-b6bc-6b8339a0a8bc + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: A/B choice randomized (choice order reversed) reference: '' - ef78f10f-aadf-4cc0-9c63-7222d63796c6: !Template + f9f889b4-4789-4529-9792-ef09b3b2544c: !Template answer_choices: null - id: ef78f10f-aadf-4cc0-9c63-7222d63796c6 + id: f9f889b4-4789-4529-9792-ef09b3b2544c jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -219,17 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 - reference: '' - f7d89f75-6fcf-42be-a1a5-0ac49184f282: !Template - answer_choices: Yes ||| No - id: f7d89f75-6fcf-42be-a1a5-0ac49184f282 - jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] - }} - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences + name: random_choice_A_B reference: '' diff --git a/promptsource/templates/blimp/wh_vs_that_no_gap_long_distance/templates.yaml b/promptsource/templates/blimp/wh_vs_that_no_gap_long_distance/templates.yaml index 8820974dd..eeb3b0a6e 100644 --- a/promptsource/templates/blimp/wh_vs_that_no_gap_long_distance/templates.yaml +++ b/promptsource/templates/blimp/wh_vs_that_no_gap_long_distance/templates.yaml @@ -1,37 +1,62 @@ dataset: blimp subset: wh_vs_that_no_gap_long_distance templates: - 246ed2f2-f3e7-4a12-b0da-282e961a049f: !Template + 0d2df529-7e58-4195-ac23-401595e22fa7: !Template answer_choices: A ||| B - id: 246ed2f2-f3e7-4a12-b0da-282e961a049f - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] - }}' + id: 0d2df529-7e58-4195-ac23-401595e22fa7 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'A/B choice 2 (bad-good, answer: B)' + name: A/B choice randomized (choice order reversed) reference: '' - 373fb84b-81b1-4306-94a7-5052b4661061: !Template + 2e268d7a-96e2-4bce-a6d6-a84d42bc7661: !Template answer_choices: null - id: 373fb84b-81b1-4306-94a7-5052b4661061 + id: 2e268d7a-96e2-4bce-a6d6-a84d42bc7661 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -39,11 +64,11 @@ templates: {% if shuffled_order == 0 %} - 1 + A {% else %} - 2 + B {% endif %}' metadata: !TemplateMetadata @@ -51,14 +76,14 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: random_choice_A_B_2 reference: '' - 4963c845-8818-4201-98a7-7752d67f0eb6: !Template + 5b1f54d8-5325-4314-bfcd-518753e52762: !Template answer_choices: null - id: 4963c845-8818-4201-98a7-7752d67f0eb6 + id: 5b1f54d8-5325-4314-bfcd-518753e52762 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -90,89 +115,140 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: random_choice_1_2_2 reference: '' - 7f680752-38cf-4ef6-91b1-9ce13f2ff6e2: !Template - answer_choices: B ||| A - id: 7f680752-38cf-4ef6-91b1-9ce13f2ff6e2 - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] - }}' + 68845ea6-10ad-400b-b680-a1aa576a1059: !Template + answer_choices: Yes ||| No + id: 68845ea6-10ad-400b-b680-a1aa576a1059 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'B/A choice, 2 (bad-good, answer: B)' + original_task: false + name: Yes/No binary judgment for good sentences (choice order reversed) reference: '' - c044e8d3-cb02-40ad-a34c-93fa78c99dda: !Template - answer_choices: B ||| A - id: c044e8d3-cb02-40ad-a34c-93fa78c99dda - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] - }}' + 80f08dc9-709a-4a1c-a741-aba516eb3bb3: !Template + answer_choices: Yes ||| No + id: 80f08dc9-709a-4a1c-a741-aba516eb3bb3 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'B/A choice 1 (good-bad, answer: A)' + original_task: false + name: Yes/No binary judgment for good sentences reference: '' - d6599adf-d39a-497f-abc1-a19a718dc5b5: !Template + 94016858-104f-4181-b395-d7ed6740b4a4: !Template answer_choices: Yes ||| No - id: d6599adf-d39a-497f-abc1-a19a718dc5b5 - jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] - }} + id: 94016858-104f-4181-b395-d7ed6740b4a4 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata - choices_in_prompt: false + choices_in_prompt: true metrics: - Accuracy original_task: false - name: Yes/No binary judgment for good sentences + name: Yes/No binary judgment for bad sentences (choice order reversed) reference: '' - d79f19df-ff07-44f5-ae88-6d89322da1d9: !Template - answer_choices: A ||| B - id: d79f19df-ff07-44f5-ae88-6d89322da1d9 - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] - }}' + 94282ca6-d70a-452f-a8d3-24d12d5b6e41: !Template + answer_choices: null + id: 94282ca6-d70a-452f-a8d3-24d12d5b6e41 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'A/B choice 1 (good-bad, answer: A)' + name: random_choice_1_2 reference: '' - ddffc67a-181e-4716-a465-17ab9db85e22: !Template - answer_choices: null - id: ddffc67a-181e-4716-a465-17ab9db85e22 + 9560095e-3b90-450a-a0b2-a18b6e9659a8: !Template + answer_choices: Yes ||| No + id: 9560095e-3b90-450a-a0b2-a18b6e9659a8 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences + reference: '' + afe6553a-98e8-4b0a-b59c-91e1705fa192: !Template + answer_choices: A ||| B + id: afe6553a-98e8-4b0a-b59c-91e1705fa192 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} - {% if shuffled_order == 0 %} - - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -180,11 +256,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: A/B choice randomized reference: '' - f0a06b70-e486-4d97-a378-1c9984ce7e00: !Template + e06fd840-fb46-428a-855c-554c50e5645e: !Template answer_choices: null - id: f0a06b70-e486-4d97-a378-1c9984ce7e00 + id: e06fd840-fb46-428a-855c-554c50e5645e jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer A or B. @@ -221,15 +297,3 @@ templates: original_task: true name: random_choice_A_B reference: '' - f61f57ef-8bba-4be1-8e3b-4148239ca7d5: !Template - answer_choices: Yes ||| No - id: f61f57ef-8bba-4be1-8e3b-4148239ca7d5 - jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] - }} - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences - reference: '' diff --git a/promptsource/templates/blimp/wh_vs_that_with_gap/templates.yaml b/promptsource/templates/blimp/wh_vs_that_with_gap/templates.yaml index c78e00d86..84b6abb47 100644 --- a/promptsource/templates/blimp/wh_vs_that_with_gap/templates.yaml +++ b/promptsource/templates/blimp/wh_vs_that_with_gap/templates.yaml @@ -1,9 +1,25 @@ dataset: blimp subset: wh_vs_that_with_gap templates: - 017b9ef0-83b1-46e6-ab86-747e9f242d28: !Template + 5caeb435-b9b0-48c5-a7a1-16cad62d65f3: !Template + answer_choices: Yes ||| No + id: 5caeb435-b9b0-48c5-a7a1-16cad62d65f3 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences (choice order reversed) + reference: '' + 6477dd7a-0af8-4142-9d72-7ec0fdf54b1d: !Template answer_choices: null - id: 017b9ef0-83b1-46e6-ab86-747e9f242d28 + id: 6477dd7a-0af8-4142-9d72-7ec0fdf54b1d jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer 1 or 2. @@ -40,61 +56,67 @@ templates: original_task: true name: random_choice_1_2 reference: '' - 3bce33c7-3fd7-4642-a5fd-f03ea801fd48: !Template + b973374b-1ba6-4472-8552-f458844c7f53: !Template answer_choices: Yes ||| No - id: 3bce33c7-3fd7-4642-a5fd-f03ea801fd48 - jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] - }} + id: b973374b-1ba6-4472-8552-f458844c7f53 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata - choices_in_prompt: false + choices_in_prompt: true metrics: - Accuracy original_task: false - name: Yes/No binary judgment for good sentences + name: Yes/No binary judgment for good sentences (choice order reversed) reference: '' - 3d7ea5ae-840e-4778-a53b-79ea82d90648: !Template - answer_choices: A ||| B - id: 3d7ea5ae-840e-4778-a53b-79ea82d90648 - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] - }}' + be01a6d1-3bdf-4eb8-81d4-40e6bb8d8e9d: !Template + answer_choices: Yes ||| No + id: be01a6d1-3bdf-4eb8-81d4-40e6bb8d8e9d + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'A/B choice 2 (bad-good, answer: B)' + original_task: false + name: Yes/No binary judgment for bad sentences reference: '' - 551e7c19-5c94-4ec8-b66d-751b37314489: !Template - answer_choices: null - id: 551e7c19-5c94-4ec8-b66d-751b37314489 + bff82d91-3194-48eb-ab61-3f5a4a53fdf5: !Template + answer_choices: A ||| B + id: bff82d91-3194-48eb-ab61-3f5a4a53fdf5 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -102,36 +124,27 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: A/B choice randomized reference: '' - 57cd4af1-97ab-49b2-8be6-771272cd2793: !Template + c9da5037-e20a-44f8-94bc-df3c3014df13: !Template answer_choices: Yes ||| No - id: 57cd4af1-97ab-49b2-8be6-771272cd2793 - jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] - }} - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences - reference: '' - 6675a3b5-3e1b-491f-9b36-cdec1d162230: !Template - answer_choices: A ||| B - id: 6675a3b5-3e1b-491f-9b36-cdec1d162230 - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] - }}' + id: c9da5037-e20a-44f8-94bc-df3c3014df13 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'A/B choice 1 (good-bad, answer: A)' + original_task: false + name: Yes/No binary judgment for good sentences reference: '' - 6d3ae668-08f6-49b8-ade8-8dab0a52af75: !Template + d8a2fd2f-9658-46eb-bc34-74fa8c5c17e0: !Template answer_choices: null - id: 6d3ae668-08f6-49b8-ade8-8dab0a52af75 + id: d8a2fd2f-9658-46eb-bc34-74fa8c5c17e0 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer A or B. @@ -168,35 +181,48 @@ templates: original_task: true name: random_choice_A_B_2 reference: '' - af9ec49c-76d8-41ec-86d3-c855726de0e4: !Template - answer_choices: B ||| A - id: af9ec49c-76d8-41ec-86d3-c855726de0e4 - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'B/A choice, 2 (bad-good, answer: B)' - reference: '' - cfc1dddb-d794-4a0e-921f-085c9fb1d8db: !Template - answer_choices: B ||| A - id: cfc1dddb-d794-4a0e-921f-085c9fb1d8db - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] - }}' + e6a4fc36-9684-44a0-83d0-d70a12c51ba1: !Template + answer_choices: null + id: e6a4fc36-9684-44a0-83d0-d70a12c51ba1 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'B/A choice 1 (good-bad, answer: A)' + name: random_choice_A_B reference: '' - d0a0cbd1-cb36-4b0e-988b-655bd8fa1209: !Template + ece76692-95b0-41a4-acc5-ebb031f8dd35: !Template answer_choices: null - id: d0a0cbd1-cb36-4b0e-988b-655bd8fa1209 + id: ece76692-95b0-41a4-acc5-ebb031f8dd35 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -233,3 +259,41 @@ templates: original_task: true name: random_choice_1_2_2 reference: '' + f3cd7d54-90f3-4303-83dc-90606680df99: !Template + answer_choices: A ||| B + id: f3cd7d54-90f3-4303-83dc-90606680df99 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: A/B choice randomized (choice order reversed) + reference: '' diff --git a/promptsource/templates/blimp/wh_vs_that_with_gap_long_distance/templates.yaml b/promptsource/templates/blimp/wh_vs_that_with_gap_long_distance/templates.yaml index 0794fb82e..b6aaec34b 100644 --- a/promptsource/templates/blimp/wh_vs_that_with_gap_long_distance/templates.yaml +++ b/promptsource/templates/blimp/wh_vs_that_with_gap_long_distance/templates.yaml @@ -1,36 +1,35 @@ dataset: blimp subset: wh_vs_that_with_gap_long_distance templates: - 1bf4224c-3cd2-4df1-ab50-f966ffa2a7a4: !Template - answer_choices: null - id: 1bf4224c-3cd2-4df1-ab50-f966ffa2a7a4 + 2914cfe5-91ea-40ec-87a7-b812d80ae297: !Template + answer_choices: A ||| B + id: 2914cfe5-91ea-40ec-87a7-b812d80ae297 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -38,24 +37,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: A/B choice randomized (choice order reversed) reference: '' - 22bbdd4a-1ad9-42b9-a4e0-33c346bd3fef: !Template - answer_choices: B ||| A - id: 22bbdd4a-1ad9-42b9-a4e0-33c346bd3fef - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[0] - }}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'B/A choice 1 (good-bad, answer: A)' - reference: '' - 4c34df87-6349-485c-95bf-b62faaec20c8: !Template + 30f395ce-ebb1-4028-9006-1662a1958ca9: !Template answer_choices: null - id: 4c34df87-6349-485c-95bf-b62faaec20c8 + id: 30f395ce-ebb1-4028-9006-1662a1958ca9 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -92,12 +78,12 @@ templates: original_task: true name: random_choice_1_2_2 reference: '' - b0a642c6-99a5-4317-bbbd-9fef33b8f613: !Template + 3c00e58c-042a-4710-b3db-f94b2d61828b: !Template answer_choices: null - id: b0a642c6-99a5-4317-bbbd-9fef33b8f613 + id: 3c00e58c-042a-4710-b3db-f94b2d61828b jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -129,26 +115,26 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: random_choice_A_B reference: '' - be12b2b2-8884-448e-90ed-1df65269119b: !Template + 432b04b6-abaf-4d98-afeb-e4deb674c8f6: !Template answer_choices: null - id: be12b2b2-8884-448e-90ed-1df65269119b + id: 432b04b6-abaf-4d98-afeb-e4deb674c8f6 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -156,11 +142,11 @@ templates: {% if shuffled_order == 0 %} - 1 + A {% else %} - 2 + B {% endif %}' metadata: !TemplateMetadata @@ -168,66 +154,144 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: random_choice_A_B_2 reference: '' - d1356cc1-f721-4d33-be47-9355043c0fe0: !Template + 46e50db4-324a-4dca-ae68-af8f4bf7156d: !Template answer_choices: A ||| B - id: d1356cc1-f721-4d33-be47-9355043c0fe0 - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[1] - }}' + id: 46e50db4-324a-4dca-ae68-af8f4bf7156d + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'A/B choice 2 (bad-good, answer: B)' + name: A/B choice randomized reference: '' - ddd157be-6df7-4be8-9864-3638ec9eae93: !Template + 5feecac8-705d-447c-a2f8-8f0986a73f60: !Template answer_choices: Yes ||| No - id: ddd157be-6df7-4be8-9864-3638ec9eae93 - jinja: Is this a good sentence of English? "{{ sentence_bad }}" ||| {{ answer_choices[1] - }} + id: 5feecac8-705d-447c-a2f8-8f0986a73f60 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata - choices_in_prompt: false + choices_in_prompt: true metrics: - Accuracy original_task: false name: Yes/No binary judgment for bad sentences reference: '' - e0b75387-7860-433c-a08b-02fd87172c60: !Template - answer_choices: A ||| B - id: e0b75387-7860-433c-a08b-02fd87172c60 - jinja: 'I have two sentences, Sentence A: {{ sentence_good }}, Sentence B: {{ - sentence_bad }}. Which one is a better sentence of English, A or B? ||| {{ answer_choices[0] - }}' + 84a826ff-a341-40a5-978a-db148ddaef96: !Template + answer_choices: Yes ||| No + id: 84a826ff-a341-40a5-978a-db148ddaef96 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'A/B choice 1 (good-bad, answer: A)' + original_task: false + name: Yes/No binary judgment for good sentences (choice order reversed) reference: '' - ea8e839d-6fe0-4a37-9f9a-83495102a67e: !Template - answer_choices: B ||| A - id: ea8e839d-6fe0-4a37-9f9a-83495102a67e - jinja: 'I have two sentences, Sentence A: {{ sentence_bad }}, Sentence B: {{ sentence_good - }}. Which one is a better sentence of English, B or A? ||| {{ answer_choices[1] - }}' + 8cdd2963-9f51-4f29-891d-36193ddf7617: !Template + answer_choices: null + id: 8cdd2963-9f51-4f29-891d-36193ddf7617 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: true - name: 'B/A choice, 2 (bad-good, answer: B)' + name: random_choice_1_2 reference: '' - fe404484-5c73-4187-acb7-c7e79c2da2e4: !Template + e3311cac-ad90-4a9a-b9b9-24530f9af0c9: !Template answer_choices: Yes ||| No - id: fe404484-5c73-4187-acb7-c7e79c2da2e4 - jinja: Is this a good sentence of English? "{{ sentence_good }}" ||| {{ answer_choices[0] - }} + id: e3311cac-ad90-4a9a-b9b9-24530f9af0c9 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata - choices_in_prompt: false + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: Yes/No binary judgment for bad sentences (choice order reversed) + reference: '' + f8a750a3-a7cc-479a-b4bc-6f14959c9225: !Template + answer_choices: Yes ||| No + id: f8a750a3-a7cc-479a-b4bc-6f14959c9225 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true metrics: - Accuracy original_task: false From 4f7f515523f6b65da8c064b09e6ba0e5da1bf0cd Mon Sep 17 00:00:00 2001 From: najoungkim Date: Tue, 26 Apr 2022 12:05:50 -0400 Subject: [PATCH 11/16] Minor changes to prompt names --- .../blimp/adjunct_island/templates.yaml | 12 +- .../anaphor_gender_agreement/templates.yaml | 188 ++++++------- .../anaphor_number_agreement/templates.yaml | 182 ++++++------- .../animate_subject_passive/templates.yaml | 176 ++++++------ .../animate_subject_trans/templates.yaml | 222 +++++++-------- .../templates/blimp/causative/templates.yaml | 192 ++++++------- .../blimp/complex_NP_island/templates.yaml | 236 ++++++++-------- .../templates.yaml | 164 ++++++------ .../templates.yaml | 136 +++++----- .../templates.yaml | 216 +++++++-------- .../templates.yaml | 198 +++++++------- .../templates.yaml | 252 +++++++++--------- .../templates.yaml | 226 ++++++++-------- .../templates.yaml | 210 +++++++-------- .../templates.yaml | 154 +++++------ .../templates.yaml | 170 ++++++------ .../templates.yaml | 216 +++++++-------- .../templates.yaml | 226 ++++++++-------- .../templates.yaml | 220 +++++++-------- .../blimp/drop_argument/templates.yaml | 192 ++++++------- .../blimp/ellipsis_n_bar_1/templates.yaml | 194 +++++++------- .../blimp/ellipsis_n_bar_2/templates.yaml | 176 ++++++------ .../templates.yaml | 206 +++++++------- .../templates.yaml | 198 +++++++------- .../templates.yaml | 214 +++++++-------- .../templates.yaml | 204 +++++++------- .../templates.yaml | 164 ++++++------ .../templates/blimp/inchoative/templates.yaml | 156 +++++------ .../blimp/intransitive/templates.yaml | 210 +++++++-------- .../templates.yaml | 216 +++++++-------- .../templates.yaml | 168 ++++++------ .../templates.yaml | 200 +++++++------- .../templates.yaml | 154 +++++------ .../templates.yaml | 244 ++++++++--------- .../templates.yaml | 180 ++++++------- .../templates.yaml | 200 +++++++------- .../blimp/npi_present_1/templates.yaml | 182 ++++++------- .../blimp/npi_present_2/templates.yaml | 218 +++++++-------- .../only_npi_licensor_present/templates.yaml | 222 +++++++-------- .../blimp/only_npi_scope/templates.yaml | 192 ++++++------- .../templates/blimp/passive_1/templates.yaml | 184 ++++++------- .../templates/blimp/passive_2/templates.yaml | 190 ++++++------- .../principle_A_c_command/templates.yaml | 208 +++++++-------- .../blimp/principle_A_case_1/templates.yaml | 202 +++++++------- .../blimp/principle_A_case_2/templates.yaml | 216 +++++++-------- .../blimp/principle_A_domain_1/templates.yaml | 156 +++++------ .../blimp/principle_A_domain_2/templates.yaml | 172 ++++++------ .../blimp/principle_A_domain_3/templates.yaml | 216 +++++++-------- .../principle_A_reconstruction/templates.yaml | 198 +++++++------- .../templates.yaml | 206 +++++++------- .../templates.yaml | 230 ++++++++-------- .../templates.yaml | 174 ++++++------ .../templates.yaml | 212 +++++++-------- .../sentential_subject_island/templates.yaml | 198 +++++++------- .../superlative_quantifiers_1/templates.yaml | 184 ++++++------- .../superlative_quantifiers_2/templates.yaml | 182 ++++++------- .../blimp/tough_vs_raising_1/templates.yaml | 192 ++++++------- .../blimp/tough_vs_raising_2/templates.yaml | 214 +++++++-------- .../templates/blimp/transitive/templates.yaml | 178 ++++++------- .../templates/blimp/wh_island/templates.yaml | 206 +++++++------- .../wh_questions_object_gap/templates.yaml | 202 +++++++------- .../wh_questions_subject_gap/templates.yaml | 228 ++++++++-------- .../templates.yaml | 236 ++++++++-------- .../blimp/wh_vs_that_no_gap/templates.yaml | 186 ++++++------- .../templates.yaml | 192 ++++++------- .../blimp/wh_vs_that_with_gap/templates.yaml | 198 +++++++------- .../templates.yaml | 142 +++++----- 67 files changed, 6494 insertions(+), 6494 deletions(-) diff --git a/promptsource/templates/blimp/adjunct_island/templates.yaml b/promptsource/templates/blimp/adjunct_island/templates.yaml index 1a56a5ce8..ead81d2e5 100644 --- a/promptsource/templates/blimp/adjunct_island/templates.yaml +++ b/promptsource/templates/blimp/adjunct_island/templates.yaml @@ -37,7 +37,7 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized (choice order reversed) + name: 'A/B choice randomized (choice order: B-A)' reference: '' 6e71cd4b-882c-4458-9cd4-fd83f849ad09: !Template answer_choices: null @@ -131,7 +131,7 @@ templates: metrics: - Accuracy original_task: false - name: Yes/No binary judgment for bad sentences (choice order reversed) + name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' reference: '' 8f354d74-5ffb-4cbc-b119-b79f2db00df3: !Template answer_choices: Yes ||| No @@ -147,7 +147,7 @@ templates: metrics: - Accuracy original_task: false - name: Yes/No binary judgment for bad sentences + name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' reference: '' 967ca4e2-162f-4365-a1f5-2c87c0e874ce: !Template answer_choices: null @@ -202,7 +202,7 @@ templates: metrics: - Accuracy original_task: false - name: Yes/No binary judgment for good sentences + name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' reference: '' c45d5bac-968e-4e39-96b7-d18a5ddb225c: !Template answer_choices: A ||| B @@ -240,7 +240,7 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized + name: 'A/B choice randomized (choice order: A-B)' reference: '' d11a37d7-7c42-4ee0-96c5-ffd24ad7073e: !Template answer_choices: null @@ -295,5 +295,5 @@ templates: metrics: - Accuracy original_task: false - name: Yes/No binary judgment for good sentences (choice order reversed) + name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' reference: '' diff --git a/promptsource/templates/blimp/anaphor_gender_agreement/templates.yaml b/promptsource/templates/blimp/anaphor_gender_agreement/templates.yaml index 6fb689db0..f056324fc 100644 --- a/promptsource/templates/blimp/anaphor_gender_agreement/templates.yaml +++ b/promptsource/templates/blimp/anaphor_gender_agreement/templates.yaml @@ -1,50 +1,12 @@ dataset: blimp subset: anaphor_gender_agreement templates: - 10c299d4-88b2-4781-8202-accc479e5bf2: !Template - answer_choices: A ||| B - id: 10c299d4-88b2-4781-8202-accc479e5bf2 - jinja: '{% set shuffled_order = [0, 1] | random %} - - Read the following two sentences. - - {% if shuffled_order == 0 %} - - Sentence A: {{ sentence_good }} - - Sentence B: {{ sentence_bad }} - - {% else %} - - Sentence A: {{ sentence_bad }} - - Sentence B: {{ sentence_good }} - - {% endif %} - - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} - - {% endif %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: A/B choice randomized (choice order reversed) - reference: '' - 31007ad7-9fcc-431d-958b-55a9e3e85e62: !Template + 0c692382-5675-48f9-81a7-a9ef7cc18c43: !Template answer_choices: null - id: 31007ad7-9fcc-431d-958b-55a9e3e85e62 + id: 0c692382-5675-48f9-81a7-a9ef7cc18c43 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -76,11 +38,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: random_choice_A_B_2 reference: '' - 34f21662-aac6-4dc6-9bfb-7a0975cfc9c8: !Template + 162a2f7a-afaf-4af9-9a2c-c6d6dd32a89e: !Template answer_choices: A ||| B - id: 34f21662-aac6-4dc6-9bfb-7a0975cfc9c8 + id: 162a2f7a-afaf-4af9-9a2c-c6d6dd32a89e jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -114,26 +76,42 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized + name: 'A/B choice randomized (choice order: A-B)' + reference: '' + 2ae00b0c-a3b8-4adf-a828-08bdc6531814: !Template + answer_choices: Yes ||| No + id: 2ae00b0c-a3b8-4adf-a828-08bdc6531814 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' reference: '' - 47617b60-4498-4b76-bc4f-590e0cf9fddb: !Template + 2d83a989-9bf8-48fb-909a-0ac5aa334a51: !Template answer_choices: null - id: 47617b60-4498-4b76-bc4f-590e0cf9fddb + id: 2d83a989-9bf8-48fb-909a-0ac5aa334a51 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -141,11 +119,11 @@ templates: {% if shuffled_order == 0 %} - A + 1 {% else %} - B + 2 {% endif %}' metadata: !TemplateMetadata @@ -153,27 +131,27 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: random_choice_1_2 reference: '' - 5df5fcdc-81c8-4d6d-8b7e-57a0ea71d83c: !Template + 9aae2f9d-51f8-4388-a3b6-3014bd9ee634: !Template answer_choices: Yes ||| No - id: 5df5fcdc-81c8-4d6d-8b7e-57a0ea71d83c + id: 9aae2f9d-51f8-4388-a3b6-3014bd9ee634 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: Yes/No binary judgment for good sentences + name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' reference: '' - 6a81d250-ca60-4bd7-ad38-60f42a3e68bf: !Template + bdfe7076-7af9-454d-8984-3b8a8f5f96e9: !Template answer_choices: Yes ||| No - id: 6a81d250-ca60-4bd7-ad38-60f42a3e68bf + id: bdfe7076-7af9-454d-8984-3b8a8f5f96e9 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -185,38 +163,37 @@ templates: metrics: - Accuracy original_task: false - name: Yes/No binary judgment for good sentences (choice order reversed) + name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' reference: '' - 6ddd2ea0-be02-4704-ab24-568cc4169776: !Template - answer_choices: null - id: 6ddd2ea0-be02-4704-ab24-568cc4169776 + cfbafd1a-03b6-41bd-a8cc-2ba4ccc480c3: !Template + answer_choices: A ||| B + id: cfbafd1a-03b6-41bd-a8cc-2ba4ccc480c3 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -224,43 +201,66 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: 'A/B choice randomized (choice order: B-A)' reference: '' - 710e6025-416c-4118-a2e0-d732ed9559a4: !Template + d2945120-7955-4217-8d8f-3c8f824b307e: !Template answer_choices: Yes ||| No - id: 710e6025-416c-4118-a2e0-d732ed9559a4 + id: d2945120-7955-4217-8d8f-3c8f824b307e jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: Yes/No binary judgment for bad sentences (choice order reversed) + name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' reference: '' - a5be9eb3-ce3c-4228-a764-6630b5d89944: !Template - answer_choices: Yes ||| No - id: a5be9eb3-ce3c-4228-a764-6630b5d89944 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + e1ede558-4086-4430-8cea-3b0b86c1e8b2: !Template + answer_choices: null + id: e1ede558-4086-4430-8cea-3b0b86c1e8b2 + jinja: '{% set shuffled_order = [0, 1] | random %} - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences + original_task: true + name: random_choice_A_B reference: '' - efc454a6-899f-48a0-846c-de78fe3499da: !Template + ebe1dc30-087d-45ba-941b-8d7b43083e0d: !Template answer_choices: null - id: efc454a6-899f-48a0-846c-de78fe3499da + id: ebe1dc30-087d-45ba-941b-8d7b43083e0d jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. diff --git a/promptsource/templates/blimp/anaphor_number_agreement/templates.yaml b/promptsource/templates/blimp/anaphor_number_agreement/templates.yaml index c30fc2455..db985435c 100644 --- a/promptsource/templates/blimp/anaphor_number_agreement/templates.yaml +++ b/promptsource/templates/blimp/anaphor_number_agreement/templates.yaml @@ -1,25 +1,25 @@ dataset: blimp subset: anaphor_number_agreement templates: - 0b65db41-53fc-4147-b1f3-e70a0095d57d: !Template + 0b145f73-63d8-4d19-8aa7-7f69909ac2e1: !Template answer_choices: Yes ||| No - id: 0b65db41-53fc-4147-b1f3-e70a0095d57d + id: 0b145f73-63d8-4d19-8aa7-7f69909ac2e1 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: Yes/No binary judgment for bad sentences (choice order reversed) + name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' reference: '' - 384a6063-63cd-4fd1-85db-1d0e0b64f648: !Template + 1033ad24-1228-462a-97bb-e84dd3f484c1: !Template answer_choices: A ||| B - id: 384a6063-63cd-4fd1-85db-1d0e0b64f648 + id: 1033ad24-1228-462a-97bb-e84dd3f484c1 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -53,11 +53,27 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized + name: 'A/B choice randomized (choice order: A-B)' + reference: '' + 165fdaa5-19fc-49d3-84ab-657c1f108dc8: !Template + answer_choices: Yes ||| No + id: 165fdaa5-19fc-49d3-84ab-657c1f108dc8 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' reference: '' - 77503d37-b2e0-44db-b9a1-00c6f9cabed7: !Template + 1d973777-1811-4c8b-8854-e6473263e524: !Template answer_choices: null - id: 77503d37-b2e0-44db-b9a1-00c6f9cabed7 + id: 1d973777-1811-4c8b-8854-e6473263e524 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer A or B. @@ -94,12 +110,12 @@ templates: original_task: true name: random_choice_A_B reference: '' - 7e77764d-bb84-4079-bcf4-e63e0a7909ea: !Template + 4aa5cd48-6914-43bb-917f-962b4cf1303a: !Template answer_choices: null - id: 7e77764d-bb84-4079-bcf4-e63e0a7909ea + id: 4aa5cd48-6914-43bb-917f-962b4cf1303a jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -131,11 +147,27 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: random_choice_1_2 reference: '' - 82450d0b-0e57-4dd8-94d5-d79351b882dc: !Template + 576c4938-8ff8-40d9-b7fc-b44c866ba1d1: !Template + answer_choices: Yes ||| No + id: 576c4938-8ff8-40d9-b7fc-b44c866ba1d1 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' + reference: '' + 8c566a9d-fbe5-4087-b672-b83116ca017c: !Template answer_choices: null - id: 82450d0b-0e57-4dd8-94d5-d79351b882dc + id: 8c566a9d-fbe5-4087-b672-b83116ca017c jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer A or B. @@ -172,36 +204,51 @@ templates: original_task: true name: random_choice_A_B_2 reference: '' - 8c3bd71c-646b-440e-9f16-579f6d88af02: !Template - answer_choices: null - id: 8c3bd71c-646b-440e-9f16-579f6d88af02 + 9fc3ec60-d67f-47af-ab3a-d19f2ad9f78b: !Template + answer_choices: Yes ||| No + id: 9fc3ec60-d67f-47af-ab3a-d19f2ad9f78b + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' + reference: '' + cca53f6a-1f7c-4b2a-bae6-bab87923ec8d: !Template + answer_choices: A ||| B + id: cca53f6a-1f7c-4b2a-bae6-bab87923ec8d jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -209,37 +256,38 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: 'A/B choice randomized (choice order: B-A)' reference: '' - ac349cca-b921-4dbe-8a01-38e924a924d0: !Template - answer_choices: A ||| B - id: ac349cca-b921-4dbe-8a01-38e924a924d0 + e379133e-23b1-4ce8-a978-3f3027fea41b: !Template + answer_choices: null + id: e379133e-23b1-4ce8-a978-3f3027fea41b jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| - {{ answer_choices[0] }} + {% if shuffled_order == 0 %} + + 1 {% else %} - {{ answer_choices[1] }} + 2 {% endif %}' metadata: !TemplateMetadata @@ -247,53 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized (choice order reversed) - reference: '' - ca01676d-f0f8-42e0-9ab5-41031bfed413: !Template - answer_choices: Yes ||| No - id: ca01676d-f0f8-42e0-9ab5-41031bfed413 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences - reference: '' - cb1f9887-ee16-40c5-8421-3021fa8dd287: !Template - answer_choices: Yes ||| No - id: cb1f9887-ee16-40c5-8421-3021fa8dd287 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences (choice order reversed) - reference: '' - faa27b3f-2c13-4a30-bcf9-71a5a7a3f129: !Template - answer_choices: Yes ||| No - id: faa27b3f-2c13-4a30-bcf9-71a5a7a3f129 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences + name: random_choice_1_2_2 reference: '' diff --git a/promptsource/templates/blimp/animate_subject_passive/templates.yaml b/promptsource/templates/blimp/animate_subject_passive/templates.yaml index f003eaa40..7b2eadcea 100644 --- a/promptsource/templates/blimp/animate_subject_passive/templates.yaml +++ b/promptsource/templates/blimp/animate_subject_passive/templates.yaml @@ -1,9 +1,25 @@ dataset: blimp subset: animate_subject_passive templates: - 0c63ccdb-f4c4-48b7-8bff-9ef98e874c11: !Template + 453b7cec-e022-4754-8227-f1b16412f82d: !Template + answer_choices: Yes ||| No + id: 453b7cec-e022-4754-8227-f1b16412f82d + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' + reference: '' + 64a41fa3-dbf6-4082-94f0-c23b58df709e: !Template answer_choices: A ||| B - id: 0c63ccdb-f4c4-48b7-8bff-9ef98e874c11 + id: 64a41fa3-dbf6-4082-94f0-c23b58df709e jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -22,7 +38,7 @@ templates: {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order + Which one is a better sentence of English, A or B? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -37,11 +53,11 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized (choice order reversed) + name: 'A/B choice randomized (choice order: A-B)' reference: '' - 453cf076-9c83-474c-bc81-b8cba3668e85: !Template + 7a84023e-edd0-4b33-88ac-295fed2391e9: !Template answer_choices: null - id: 453cf076-9c83-474c-bc81-b8cba3668e85 + id: 7a84023e-edd0-4b33-88ac-295fed2391e9 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -78,40 +94,24 @@ templates: original_task: true name: random_choice_1_2_2 reference: '' - 47126c4b-302d-47e6-8605-f2765c678bb8: !Template - answer_choices: Yes ||| No - id: 47126c4b-302d-47e6-8605-f2765c678bb8 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences (choice order reversed) - reference: '' - 48567d36-352c-41bb-a25b-282ffae6c1e1: !Template + 881cfd60-e91a-4314-a235-6196cd6ff251: !Template answer_choices: null - id: 48567d36-352c-41bb-a25b-282ffae6c1e1 + id: 881cfd60-e91a-4314-a235-6196cd6ff251 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -119,11 +119,11 @@ templates: {% if shuffled_order == 0 %} - 1 + A {% else %} - 2 + B {% endif %}' metadata: !TemplateMetadata @@ -131,53 +131,54 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: random_choice_A_B_2 reference: '' - b561a033-3b01-4aca-9ec4-9641e54a5fde: !Template + 97cb7e83-2385-461e-bc05-dc5511aaff5f: !Template answer_choices: Yes ||| No - id: b561a033-3b01-4aca-9ec4-9641e54a5fde + id: 97cb7e83-2385-461e-bc05-dc5511aaff5f jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: Yes/No binary judgment for bad sentences (choice order reversed) + name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' reference: '' - b724ca94-7643-46c7-a64d-8a86b04fd48c: !Template - answer_choices: A ||| B - id: b724ca94-7643-46c7-a64d-8a86b04fd48c + 9a2eb37a-8256-4607-90c4-14ba522942b9: !Template + answer_choices: null + id: 9a2eb37a-8256-4607-90c4-14ba522942b9 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| - {{ answer_choices[0] }} + {% if shuffled_order == 0 %} + + 1 {% else %} - {{ answer_choices[1] }} + 2 {% endif %}' metadata: !TemplateMetadata @@ -185,15 +186,31 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized + name: random_choice_1_2 reference: '' - cd2d6e9c-bc1b-4f43-8633-f2dc6f4845fc: !Template + b89dfc16-3601-4ab7-9275-0be0475d1f57: !Template answer_choices: Yes ||| No - id: cd2d6e9c-bc1b-4f43-8633-f2dc6f4845fc + id: b89dfc16-3601-4ab7-9275-0be0475d1f57 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' + reference: '' + b985ea33-3169-4d14-9727-1c1e1681d33c: !Template + answer_choices: Yes ||| No + id: b985ea33-3169-4d14-9727-1c1e1681d33c + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata @@ -201,14 +218,14 @@ templates: metrics: - Accuracy original_task: false - name: Yes/No binary judgment for bad sentences + name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' reference: '' - cf18d556-acc0-4675-a28a-39deceff0cbc: !Template + d1b2916a-daee-4424-9238-d7b40ca79aaa: !Template answer_choices: null - id: cf18d556-acc0-4675-a28a-39deceff0cbc + id: d1b2916a-daee-4424-9238-d7b40ca79aaa jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -240,54 +257,37 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 - reference: '' - d9bdf45e-6574-4a1f-8bc6-16edc2c118a1: !Template - answer_choices: Yes ||| No - id: d9bdf45e-6574-4a1f-8bc6-16edc2c118a1 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences + name: random_choice_A_B reference: '' - e55eb95e-29e1-4459-9782-58fba8b03ffc: !Template - answer_choices: null - id: e55eb95e-29e1-4459-9782-58fba8b03ffc + ee24cebd-6a6f-4c39-8fa2-ec58545dcc88: !Template + answer_choices: A ||| B + id: ee24cebd-6a6f-4c39-8fa2-ec58545dcc88 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -295,5 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: 'A/B choice randomized (choice order: B-A)' reference: '' diff --git a/promptsource/templates/blimp/animate_subject_trans/templates.yaml b/promptsource/templates/blimp/animate_subject_trans/templates.yaml index 3dc0a4fbe..c51a25f15 100644 --- a/promptsource/templates/blimp/animate_subject_trans/templates.yaml +++ b/promptsource/templates/blimp/animate_subject_trans/templates.yaml @@ -1,36 +1,51 @@ dataset: blimp subset: animate_subject_trans templates: - 185a0721-f8b5-49b8-9101-b97a70374247: !Template - answer_choices: null - id: 185a0721-f8b5-49b8-9101-b97a70374247 + 073e4b7b-95bb-4dfe-849d-7b97e58ee49b: !Template + answer_choices: Yes ||| No + id: 073e4b7b-95bb-4dfe-849d-7b97e58ee49b + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' + reference: '' + 304746e1-9763-43c3-b17a-273c149c4dbe: !Template + answer_choices: A ||| B + id: 304746e1-9763-43c3-b17a-273c149c4dbe jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -38,46 +53,14 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: 'A/B choice randomized (choice order: A-B)' reference: '' - 2835e45d-c778-468c-9ee7-bbd5dc8a7a92: !Template - answer_choices: Yes ||| No - id: 2835e45d-c778-468c-9ee7-bbd5dc8a7a92 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences (choice order reversed) - reference: '' - ac557360-93dd-4dc1-a393-9c1bf3907af4: !Template - answer_choices: Yes ||| No - id: ac557360-93dd-4dc1-a393-9c1bf3907af4 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences - reference: '' - b8137185-8581-4aaa-a2f4-cc7d48e773ca: !Template + 30aeeae7-ca4c-400c-9544-2d8ed77a9816: !Template answer_choices: null - id: b8137185-8581-4aaa-a2f4-cc7d48e773ca + id: 30aeeae7-ca4c-400c-9544-2d8ed77a9816 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -109,27 +92,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 - reference: '' - c2f25416-e628-4123-a6cc-8429345fd6f0: !Template - answer_choices: Yes ||| No - id: c2f25416-e628-4123-a6cc-8429345fd6f0 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences (choice order reversed) + name: random_choice_A_B reference: '' - d3e7d4b7-6c35-450a-aafa-f9f60dffaeff: !Template + 44e124f6-1877-4f30-ae53-0858dde71199: !Template answer_choices: Yes ||| No - id: d3e7d4b7-6c35-450a-aafa-f9f60dffaeff + id: 44e124f6-1877-4f30-ae53-0858dde71199 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -141,37 +108,38 @@ templates: metrics: - Accuracy original_task: false - name: Yes/No binary judgment for bad sentences + name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' reference: '' - e1322160-dcc6-4b8a-975f-86965799505a: !Template - answer_choices: A ||| B - id: e1322160-dcc6-4b8a-975f-86965799505a + 72835b3d-b9ac-42ea-a611-16e5571244cd: !Template + answer_choices: null + id: 72835b3d-b9ac-42ea-a611-16e5571244cd jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| - {{ answer_choices[0] }} + {% if shuffled_order == 0 %} + + A {% else %} - {{ answer_choices[1] }} + B {% endif %}' metadata: !TemplateMetadata @@ -179,11 +147,11 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized + name: random_choice_A_B_2 reference: '' - ee63121a-8c10-4a19-a8da-080cc36399d5: !Template + 8d86aa5d-10fd-495b-84f3-1c60d55304f8: !Template answer_choices: null - id: ee63121a-8c10-4a19-a8da-080cc36399d5 + id: 8d86aa5d-10fd-495b-84f3-1c60d55304f8 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer 1 or 2. @@ -220,36 +188,51 @@ templates: original_task: true name: random_choice_1_2 reference: '' - f06e7379-09ca-490e-b60b-54f2bdcf9830: !Template - answer_choices: null - id: f06e7379-09ca-490e-b60b-54f2bdcf9830 + a62914b0-cc0d-47e0-ac13-8f12f99fda7e: !Template + answer_choices: Yes ||| No + id: a62914b0-cc0d-47e0-ac13-8f12f99fda7e + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' + reference: '' + a73020d6-d1bb-45d4-9dec-6c226864ef6a: !Template + answer_choices: A ||| B + id: a73020d6-d1bb-45d4-9dec-6c226864ef6a jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -257,37 +240,38 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: 'A/B choice randomized (choice order: B-A)' reference: '' - f291441f-914d-406a-adbf-8641415ea6c3: !Template - answer_choices: A ||| B - id: f291441f-914d-406a-adbf-8641415ea6c3 + c185d3df-50a5-433a-a904-c56b95e5cb29: !Template + answer_choices: null + id: c185d3df-50a5-433a-a904-c56b95e5cb29 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| - {{ answer_choices[0] }} + {% if shuffled_order == 0 %} + + 1 {% else %} - {{ answer_choices[1] }} + 2 {% endif %}' metadata: !TemplateMetadata @@ -295,5 +279,21 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized (choice order reversed) + name: random_choice_1_2_2 + reference: '' + dd656c7e-2716-4791-81a8-2e9945a4345d: !Template + answer_choices: Yes ||| No + id: dd656c7e-2716-4791-81a8-2e9945a4345d + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' reference: '' diff --git a/promptsource/templates/blimp/causative/templates.yaml b/promptsource/templates/blimp/causative/templates.yaml index d7d2ec6e7..fa2f0fab1 100644 --- a/promptsource/templates/blimp/causative/templates.yaml +++ b/promptsource/templates/blimp/causative/templates.yaml @@ -1,24 +1,24 @@ dataset: blimp subset: causative templates: - 15b8261d-8fb8-4977-8e02-ebc6c82974f7: !Template + 0bc2dedf-4691-4b25-a06a-ecb619c36087: !Template answer_choices: null - id: 15b8261d-8fb8-4977-8e02-ebc6c82974f7 + id: 0bc2dedf-4691-4b25-a06a-ecb619c36087 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -26,11 +26,11 @@ templates: {% if shuffled_order == 0 %} - A + 1 {% else %} - B + 2 {% endif %}' metadata: !TemplateMetadata @@ -38,26 +38,26 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: random_choice_1_2 reference: '' - 5b8633f8-b78f-4798-8a2d-d36d5863cc09: !Template + 3ede30de-975e-44e2-94e4-106b191d4bf4: !Template answer_choices: null - id: 5b8633f8-b78f-4798-8a2d-d36d5863cc09 + id: 3ede30de-975e-44e2-94e4-106b191d4bf4 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -65,11 +65,11 @@ templates: {% if shuffled_order == 0 %} - 1 + A {% else %} - 2 + B {% endif %}' metadata: !TemplateMetadata @@ -77,54 +77,53 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: random_choice_A_B_2 reference: '' - 8293e16e-3edd-4ff3-8e6f-c4fd24ca58a2: !Template + 8a92bd39-198d-4233-a286-932d01339249: !Template answer_choices: Yes ||| No - id: 8293e16e-3edd-4ff3-8e6f-c4fd24ca58a2 + id: 8a92bd39-198d-4233-a286-932d01339249 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: Yes/No binary judgment for bad sentences (choice order reversed) + name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' reference: '' - 97b44339-53be-4e33-a84e-4466eb5688f4: !Template - answer_choices: null - id: 97b44339-53be-4e33-a84e-4466eb5688f4 + 9585c165-4da8-4f13-8952-cb7dd8249347: !Template + answer_choices: A ||| B + id: 9585c165-4da8-4f13-8952-cb7dd8249347 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -132,43 +131,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 - reference: '' - 9bfe4827-d68e-44e1-b465-20240f9c3e1d: !Template - answer_choices: Yes ||| No - id: 9bfe4827-d68e-44e1-b465-20240f9c3e1d - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences (choice order reversed) - reference: '' - 9edceb34-f71e-48e1-9d9e-838c0b4087ec: !Template - answer_choices: Yes ||| No - id: 9edceb34-f71e-48e1-9d9e-838c0b4087ec - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences + name: 'A/B choice randomized (choice order: B-A)' reference: '' - a236ff5d-831c-4bab-a709-0a949e3497d7: !Template + 9f141a7b-43ec-44c7-8d46-10b4163ae970: !Template answer_choices: A ||| B - id: a236ff5d-831c-4bab-a709-0a949e3497d7 + id: 9f141a7b-43ec-44c7-8d46-10b4163ae970 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -202,37 +169,70 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized + name: 'A/B choice randomized (choice order: A-B)' reference: '' - a8472746-14c2-410e-8122-82610156baef: !Template - answer_choices: A ||| B - id: a8472746-14c2-410e-8122-82610156baef + a99bfe8d-3ed3-46bf-8008-87f41cab19b8: !Template + answer_choices: Yes ||| No + id: a99bfe8d-3ed3-46bf-8008-87f41cab19b8 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' + reference: '' + be157f6c-df7a-4ccf-887a-595ea99d032d: !Template + answer_choices: Yes ||| No + id: be157f6c-df7a-4ccf-887a-595ea99d032d + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' + reference: '' + d62aa60b-e51d-4353-9246-eedbfd0526dc: !Template + answer_choices: null + id: d62aa60b-e51d-4353-9246-eedbfd0526dc jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| - {{ answer_choices[0] }} + {% if shuffled_order == 0 %} + + 1 {% else %} - {{ answer_choices[1] }} + 2 {% endif %}' metadata: !TemplateMetadata @@ -240,14 +240,14 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized (choice order reversed) + name: random_choice_1_2_2 reference: '' - aa86f11c-cc83-4748-a160-d5b5e8ac7417: !Template + ed22a914-badf-4ced-a8b9-8c02c850b535: !Template answer_choices: null - id: aa86f11c-cc83-4748-a160-d5b5e8ac7417 + id: ed22a914-badf-4ced-a8b9-8c02c850b535 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -279,21 +279,21 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: random_choice_A_B reference: '' - f3fe2424-b32d-461c-9745-9e833fcb695b: !Template + f362fc21-588d-40bc-a93d-e4f9eaa1bea6: !Template answer_choices: Yes ||| No - id: f3fe2424-b32d-461c-9745-9e833fcb695b + id: f362fc21-588d-40bc-a93d-e4f9eaa1bea6 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: Yes/No binary judgment for bad sentences + name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' reference: '' diff --git a/promptsource/templates/blimp/complex_NP_island/templates.yaml b/promptsource/templates/blimp/complex_NP_island/templates.yaml index f16e65bd0..39a680bbb 100644 --- a/promptsource/templates/blimp/complex_NP_island/templates.yaml +++ b/promptsource/templates/blimp/complex_NP_island/templates.yaml @@ -1,9 +1,64 @@ dataset: blimp subset: complex_NP_island templates: - 346563c3-4bd2-4c82-83a9-8e50a158aca4: !Template + 041bbb38-1ec7-4ebd-b2dd-89a3af095309: !Template + answer_choices: Yes ||| No + id: 041bbb38-1ec7-4ebd-b2dd-89a3af095309 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' + reference: '' + 0494420d-2b79-48b3-b754-764c17dc5444: !Template + answer_choices: null + id: 0494420d-2b79-48b3-b754-764c17dc5444 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B + reference: '' + 22df7965-9604-4201-a824-daeb432efe65: !Template answer_choices: A ||| B - id: 346563c3-4bd2-4c82-83a9-8e50a158aca4 + id: 22df7965-9604-4201-a824-daeb432efe65 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -37,58 +92,26 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized (choice order reversed) - reference: '' - 3497056c-4444-44d6-be50-a66231a4f4d3: !Template - answer_choices: Yes ||| No - id: 3497056c-4444-44d6-be50-a66231a4f4d3 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences - reference: '' - 6a8a5b4b-dec7-4f23-a38c-804555c233aa: !Template - answer_choices: Yes ||| No - id: 6a8a5b4b-dec7-4f23-a38c-804555c233aa - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences + name: 'A/B choice randomized (choice order: B-A)' reference: '' - ab5d533c-ea31-483f-9c3b-1c60cb8a4d2f: !Template + 344d25cb-ed7c-4884-8c80-157dff563b2a: !Template answer_choices: null - id: ab5d533c-ea31-483f-9c3b-1c60cb8a4d2f + id: 344d25cb-ed7c-4884-8c80-157dff563b2a jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -96,11 +119,11 @@ templates: {% if shuffled_order == 0 %} - A + 1 {% else %} - B + 2 {% endif %}' metadata: !TemplateMetadata @@ -108,27 +131,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 - reference: '' - c231307c-5c87-4183-8222-077840a7aa02: !Template - answer_choices: Yes ||| No - id: c231307c-5c87-4183-8222-077840a7aa02 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences (choice order reversed) + name: random_choice_1_2 reference: '' - cdc7aef9-23ac-4993-8f60-d7080c785a04: !Template + 434e2faa-3d3e-4a3e-84fe-bdfee866b08a: !Template answer_choices: null - id: cdc7aef9-23ac-4993-8f60-d7080c785a04 + id: 434e2faa-3d3e-4a3e-84fe-bdfee866b08a jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -165,25 +172,9 @@ templates: original_task: true name: random_choice_1_2_2 reference: '' - d7f278bc-4d8a-4498-a6b5-580f1c84d7bc: !Template - answer_choices: Yes ||| No - id: d7f278bc-4d8a-4498-a6b5-580f1c84d7bc - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences (choice order reversed) - reference: '' - dc307bdb-7a73-41ca-9b0c-09736dd3d042: !Template + 6089eaff-0e6d-4507-a055-1b271d51d37b: !Template answer_choices: A ||| B - id: dc307bdb-7a73-41ca-9b0c-09736dd3d042 + id: 6089eaff-0e6d-4507-a055-1b271d51d37b jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -217,14 +208,30 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized + name: 'A/B choice randomized (choice order: A-B)' reference: '' - e852ac7d-f2ca-456e-a32d-931fcd552081: !Template + 7cd043fd-7026-428c-9b18-5416682430d0: !Template + answer_choices: Yes ||| No + id: 7cd043fd-7026-428c-9b18-5416682430d0 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' + reference: '' + 96afc9ce-f867-41b7-a52f-5e67f280b22c: !Template answer_choices: null - id: e852ac7d-f2ca-456e-a32d-931fcd552081 + id: 96afc9ce-f867-41b7-a52f-5e67f280b22c jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -256,44 +263,37 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: random_choice_A_B_2 reference: '' - fe52bb02-c6c7-4687-8d99-ed9d7bdc48d8: !Template - answer_choices: null - id: fe52bb02-c6c7-4687-8d99-ed9d7bdc48d8 - jinja: '{% set shuffled_order = [0, 1] | random %} - - Which one of the following sentences is grammatical? Please answer 1 or 2. - - {% if shuffled_order == 0 %} - - 1: {{ sentence_good }} - - 2: {{ sentence_bad }} - - {% else %} - - 1: {{ sentence_bad }} - - 2: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - 1 - - {% else %} + c7644486-bcf6-432c-98e8-a5853871ae56: !Template + answer_choices: Yes ||| No + id: c7644486-bcf6-432c-98e8-a5853871ae56 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - 2 + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' + reference: '' + e037553c-43a2-4ab4-b432-e35bcc112630: !Template + answer_choices: Yes ||| No + id: e037553c-43a2-4ab4-b432-e35bcc112630 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - {% endif %}' + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: random_choice_1_2 + original_task: false + name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' reference: '' diff --git a/promptsource/templates/blimp/coordinate_structure_constraint_complex_left_branch/templates.yaml b/promptsource/templates/blimp/coordinate_structure_constraint_complex_left_branch/templates.yaml index 7e21d99c7..734b94e3a 100644 --- a/promptsource/templates/blimp/coordinate_structure_constraint_complex_left_branch/templates.yaml +++ b/promptsource/templates/blimp/coordinate_structure_constraint_complex_left_branch/templates.yaml @@ -1,24 +1,24 @@ dataset: blimp subset: coordinate_structure_constraint_complex_left_branch templates: - 0121e488-e96b-4a16-897e-18e36d1ea63a: !Template + 21514070-d6e2-48ec-b43f-3447b8ffd989: !Template answer_choices: null - id: 0121e488-e96b-4a16-897e-18e36d1ea63a + id: 21514070-d6e2-48ec-b43f-3447b8ffd989 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -26,11 +26,11 @@ templates: {% if shuffled_order == 0 %} - A + 1 {% else %} - B + 2 {% endif %}' metadata: !TemplateMetadata @@ -38,26 +38,42 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: random_choice_1_2_2 + reference: '' + 25208006-931f-47cd-8912-1f2aa6b25875: !Template + answer_choices: Yes ||| No + id: 25208006-931f-47cd-8912-1f2aa6b25875 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' reference: '' - 36f72863-4705-4fb7-ab70-d6ac1481dde1: !Template + 25cb7f00-b9f8-4a7c-94da-3d4ca96d2fe5: !Template answer_choices: null - id: 36f72863-4705-4fb7-ab70-d6ac1481dde1 + id: 25cb7f00-b9f8-4a7c-94da-3d4ca96d2fe5 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -65,11 +81,11 @@ templates: {% if shuffled_order == 0 %} - 1 + A {% else %} - 2 + B {% endif %}' metadata: !TemplateMetadata @@ -77,11 +93,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: random_choice_A_B reference: '' - 7ac016b3-2d12-463d-9e0f-703b6cbe0a91: !Template + 2e720290-257e-4105-bcff-6ae4307f0d4f: !Template answer_choices: A ||| B - id: 7ac016b3-2d12-463d-9e0f-703b6cbe0a91 + id: 2e720290-257e-4105-bcff-6ae4307f0d4f jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -115,37 +131,38 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized (choice order reversed) + name: 'A/B choice randomized (choice order: B-A)' reference: '' - 7f50cf6a-ca05-4815-b1a3-e04f93b00ac7: !Template - answer_choices: A ||| B - id: 7f50cf6a-ca05-4815-b1a3-e04f93b00ac7 + 30e006b0-14e1-49cf-8a95-6eded8256a86: !Template + answer_choices: null + id: 30e006b0-14e1-49cf-8a95-6eded8256a86 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| - {{ answer_choices[0] }} + {% if shuffled_order == 0 %} + + 1 {% else %} - {{ answer_choices[1] }} + 2 {% endif %}' metadata: !TemplateMetadata @@ -153,38 +170,37 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized + name: random_choice_1_2 reference: '' - ade9109e-db39-4328-b259-a6ab965e5cfd: !Template - answer_choices: null - id: ade9109e-db39-4328-b259-a6ab965e5cfd + 860e79dd-3638-47fb-9959-595022d60c68: !Template + answer_choices: A ||| B + id: 860e79dd-3638-47fb-9959-595022d60c68 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -192,31 +208,31 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: 'A/B choice randomized (choice order: A-B)' reference: '' - afb4a824-cc46-45ac-bc3c-3c43d000f947: !Template + 96bedf82-1010-432e-8aa1-566ebebd6f73: !Template answer_choices: Yes ||| No - id: afb4a824-cc46-45ac-bc3c-3c43d000f947 + id: 96bedf82-1010-432e-8aa1-566ebebd6f73 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: Yes/No binary judgment for good sentences + name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' reference: '' - c0d96488-8976-4594-bc06-0cef9239081d: !Template + b08e28a2-6e94-4882-8d17-6e89c1f36cf1: !Template answer_choices: Yes ||| No - id: c0d96488-8976-4594-bc06-0cef9239081d + id: b08e28a2-6e94-4882-8d17-6e89c1f36cf1 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -224,14 +240,14 @@ templates: metrics: - Accuracy original_task: false - name: Yes/No binary judgment for good sentences (choice order reversed) + name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' reference: '' - cf445379-d6fe-4d46-86a3-8155f2101cae: !Template + c4812c70-7a21-4dab-8136-047ed2570fee: !Template answer_choices: null - id: cf445379-d6fe-4d46-86a3-8155f2101cae + id: c4812c70-7a21-4dab-8136-047ed2570fee jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -263,11 +279,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: random_choice_A_B_2 reference: '' - ea8fd446-0b09-4f76-8b49-06fb6e99e074: !Template + e11601b3-da0d-4e1f-a54e-9603a2ba6392: !Template answer_choices: Yes ||| No - id: ea8fd446-0b09-4f76-8b49-06fb6e99e074 + id: e11601b3-da0d-4e1f-a54e-9603a2ba6392 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -279,21 +295,5 @@ templates: metrics: - Accuracy original_task: false - name: Yes/No binary judgment for bad sentences (choice order reversed) - reference: '' - f861c143-c477-403a-b3ca-c051a55d374b: !Template - answer_choices: Yes ||| No - id: f861c143-c477-403a-b3ca-c051a55d374b - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences + name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' reference: '' diff --git a/promptsource/templates/blimp/coordinate_structure_constraint_object_extraction/templates.yaml b/promptsource/templates/blimp/coordinate_structure_constraint_object_extraction/templates.yaml index 6861a17e5..695e29b23 100644 --- a/promptsource/templates/blimp/coordinate_structure_constraint_object_extraction/templates.yaml +++ b/promptsource/templates/blimp/coordinate_structure_constraint_object_extraction/templates.yaml @@ -1,9 +1,9 @@ dataset: blimp subset: coordinate_structure_constraint_object_extraction templates: - 0f41865d-bfe4-4664-964d-fee06008f421: !Template + 1f3bb98a-03f9-482b-84e7-b73366fd9f92: !Template answer_choices: null - id: 0f41865d-bfe4-4664-964d-fee06008f421 + id: 1f3bb98a-03f9-482b-84e7-b73366fd9f92 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer 1 or 2. @@ -40,9 +40,9 @@ templates: original_task: true name: random_choice_1_2 reference: '' - 20db9e37-ca02-4112-abec-3923e84aa0e2: !Template + 56ffab74-f1b1-4302-9eb4-23b15809583c: !Template answer_choices: A ||| B - id: 20db9e37-ca02-4112-abec-3923e84aa0e2 + id: 56ffab74-f1b1-4302-9eb4-23b15809583c jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -61,7 +61,7 @@ templates: {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order + Which one is a better sentence of English, B or A? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -76,38 +76,37 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized + name: 'A/B choice randomized (choice order: B-A)' reference: '' - 5b38757a-3df9-4ffe-a147-3a3c6c4347ed: !Template - answer_choices: null - id: 5b38757a-3df9-4ffe-a147-3a3c6c4347ed + 82522714-9460-4ef1-a7f3-076c3e6e86c5: !Template + answer_choices: A ||| B + id: 82522714-9460-4ef1-a7f3-076c3e6e86c5 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -115,27 +114,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: 'A/B choice randomized (choice order: A-B)' reference: '' - 878a9e21-a6bc-4675-bde0-73934614f31d: !Template - answer_choices: Yes ||| No - id: 878a9e21-a6bc-4675-bde0-73934614f31d - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences - reference: '' - 8b7f1f3d-f2ac-4201-b3b5-201ef9c2bd7b: !Template + 86530617-f0eb-4aea-b5eb-32ce93b77ff4: !Template answer_choices: null - id: 8b7f1f3d-f2ac-4201-b3b5-201ef9c2bd7b + id: 86530617-f0eb-4aea-b5eb-32ce93b77ff4 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -172,9 +155,9 @@ templates: original_task: true name: random_choice_1_2_2 reference: '' - a1c8f9df-d44f-4743-9466-55b49c51382a: !Template + a1df63ad-0646-4dd2-a381-e1c9a6d2449b: !Template answer_choices: Yes ||| No - id: a1c8f9df-d44f-4743-9466-55b49c51382a + id: a1df63ad-0646-4dd2-a381-e1c9a6d2449b jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -186,46 +169,46 @@ templates: metrics: - Accuracy original_task: false - name: Yes/No binary judgment for bad sentences (choice order reversed) + name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' reference: '' - a6f76be1-a09b-4dd2-913e-6c1d9e28548c: !Template + a5cc9d03-4fc7-4d6e-9504-57e29c8f64d5: !Template answer_choices: Yes ||| No - id: a6f76be1-a09b-4dd2-913e-6c1d9e28548c + id: a5cc9d03-4fc7-4d6e-9504-57e29c8f64d5 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: Yes/No binary judgment for good sentences (choice order reversed) + name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' reference: '' - ad8ff748-0507-4f8f-857e-dfcc530e45d9: !Template + b3f9fd42-1161-4530-8ad3-88e96a9997a7: !Template answer_choices: Yes ||| No - id: ad8ff748-0507-4f8f-857e-dfcc530e45d9 + id: b3f9fd42-1161-4530-8ad3-88e96a9997a7 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: Yes/No binary judgment for bad sentences + name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' reference: '' - aecd346a-4ce9-46dd-b69c-32c02b37e076: !Template + d2fd6631-2bc3-451b-9052-42514fb1826b: !Template answer_choices: null - id: aecd346a-4ce9-46dd-b69c-32c02b37e076 + id: d2fd6631-2bc3-451b-9052-42514fb1826b jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -257,37 +240,54 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: random_choice_A_B_2 reference: '' - f8780981-6bef-41f4-bbc1-c1ecb0267bf7: !Template - answer_choices: A ||| B - id: f8780981-6bef-41f4-bbc1-c1ecb0267bf7 + e904fa30-f239-4b1c-82c6-f166b941a2cc: !Template + answer_choices: Yes ||| No + id: e904fa30-f239-4b1c-82c6-f166b941a2cc + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' + reference: '' + f6f7f7b3-dd21-4f16-bc7a-49d00278c807: !Template + answer_choices: null + id: f6f7f7b3-dd21-4f16-bc7a-49d00278c807 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| - {{ answer_choices[0] }} + {% if shuffled_order == 0 %} + + A {% else %} - {{ answer_choices[1] }} + B {% endif %}' metadata: !TemplateMetadata @@ -295,5 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized (choice order reversed) + name: random_choice_A_B reference: '' diff --git a/promptsource/templates/blimp/determiner_noun_agreement_1/templates.yaml b/promptsource/templates/blimp/determiner_noun_agreement_1/templates.yaml index 0658610e0..60c7e2e08 100644 --- a/promptsource/templates/blimp/determiner_noun_agreement_1/templates.yaml +++ b/promptsource/templates/blimp/determiner_noun_agreement_1/templates.yaml @@ -1,64 +1,9 @@ dataset: blimp subset: determiner_noun_agreement_1 templates: - 16a971e1-163d-46e5-985f-629751cc71c3: !Template - answer_choices: Yes ||| No - id: 16a971e1-163d-46e5-985f-629751cc71c3 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences - reference: '' - 23c29387-dac6-4799-b045-9984c2c4982d: !Template - answer_choices: null - id: 23c29387-dac6-4799-b045-9984c2c4982d - jinja: '{% set shuffled_order = [0, 1] | random %} - - Between the two sentences, which one is grammatical? Please answer 1 or 2. - - {% if shuffled_order == 0 %} - - 1: {{ sentence_good }} - - 2: {{ sentence_bad }} - - {% else %} - - 1: {{ sentence_bad }} - - 2: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - 1 - - {% else %} - - 2 - - {% endif %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: random_choice_1_2_2 - reference: '' - 659d61d8-c28e-4e4a-90ab-f6acb5556141: !Template + 00ea4c3a-48db-4033-b488-530f91b15f91: !Template answer_choices: A ||| B - id: 659d61d8-c28e-4e4a-90ab-f6acb5556141 + id: 00ea4c3a-48db-4033-b488-530f91b15f91 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -92,14 +37,46 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized (choice order reversed) + name: 'A/B choice randomized (choice order: B-A)' reference: '' - 6a17b03d-92e0-4d4e-9b56-49fc6be6b15c: !Template + 2dffb9b8-a687-4ec2-9de4-2756bfccd6f8: !Template + answer_choices: Yes ||| No + id: 2dffb9b8-a687-4ec2-9de4-2756bfccd6f8 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' + reference: '' + 4eaa4f27-343b-46d3-ae1f-59506600c968: !Template + answer_choices: Yes ||| No + id: 4eaa4f27-343b-46d3-ae1f-59506600c968 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' + reference: '' + 883c95e1-9c24-4dcc-b0a4-d36f791a5aec: !Template answer_choices: null - id: 6a17b03d-92e0-4d4e-9b56-49fc6be6b15c + id: 883c95e1-9c24-4dcc-b0a4-d36f791a5aec jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -131,14 +108,14 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: random_choice_1_2_2 reference: '' - 78169e75-93f7-4163-be6e-1fb4f253a6ba: !Template + 8eb177fa-7fce-4b25-8bbe-02a4e777d380: !Template answer_choices: null - id: 78169e75-93f7-4163-be6e-1fb4f253a6ba + id: 8eb177fa-7fce-4b25-8bbe-02a4e777d380 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -170,27 +147,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B - reference: '' - 8265598d-6ee0-4626-9bea-e09df0bce1f8: !Template - answer_choices: Yes ||| No - id: 8265598d-6ee0-4626-9bea-e09df0bce1f8 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences (choice order reversed) + name: random_choice_A_B_2 reference: '' - a07d22fb-6092-4448-902a-ea678f72cc32: !Template + 91c729c1-56e1-48f0-9252-912abe3ee7bf: !Template answer_choices: A ||| B - id: a07d22fb-6092-4448-902a-ea678f72cc32 + id: 91c729c1-56e1-48f0-9252-912abe3ee7bf jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -224,11 +185,50 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized + name: 'A/B choice randomized (choice order: A-B)' + reference: '' + 9fe81394-ed07-434a-9525-0e7e5b9936c6: !Template + answer_choices: null + id: 9fe81394-ed07-434a-9525-0e7e5b9936c6 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2 reference: '' - ad08907e-ae6b-48e1-80d9-efba35f9a0b8: !Template + a0883421-f6bc-4242-9142-fdb8bbd49240: !Template answer_choices: Yes ||| No - id: ad08907e-ae6b-48e1-80d9-efba35f9a0b8 + id: a0883421-f6bc-4242-9142-fdb8bbd49240 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -240,14 +240,30 @@ templates: metrics: - Accuracy original_task: false - name: Yes/No binary judgment for good sentences (choice order reversed) + name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' reference: '' - ea27c898-7d78-4f03-a96f-4f045a9f8a7f: !Template + be33a953-fbdb-41b3-896b-f750b93e97be: !Template + answer_choices: Yes ||| No + id: be33a953-fbdb-41b3-896b-f750b93e97be + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' + reference: '' + d9cf1e1e-cf34-415a-9db7-e5fee2febdaa: !Template answer_choices: null - id: ea27c898-7d78-4f03-a96f-4f045a9f8a7f + id: d9cf1e1e-cf34-415a-9db7-e5fee2febdaa jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -279,21 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 - reference: '' - f3e5a028-2344-4e21-b200-47da95bf028c: !Template - answer_choices: Yes ||| No - id: f3e5a028-2344-4e21-b200-47da95bf028c - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences + name: random_choice_A_B reference: '' diff --git a/promptsource/templates/blimp/determiner_noun_agreement_2/templates.yaml b/promptsource/templates/blimp/determiner_noun_agreement_2/templates.yaml index bad42ae27..05f49dae4 100644 --- a/promptsource/templates/blimp/determiner_noun_agreement_2/templates.yaml +++ b/promptsource/templates/blimp/determiner_noun_agreement_2/templates.yaml @@ -1,24 +1,24 @@ dataset: blimp subset: determiner_noun_agreement_2 templates: - 0339ae7d-0373-4368-b8dd-da73930d6426: !Template + 150b9c09-a4fd-410a-8481-3d29c22e0264: !Template answer_choices: null - id: 0339ae7d-0373-4368-b8dd-da73930d6426 + id: 150b9c09-a4fd-410a-8481-3d29c22e0264 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -26,11 +26,11 @@ templates: {% if shuffled_order == 0 %} - A + 1 {% else %} - B + 2 {% endif %}' metadata: !TemplateMetadata @@ -38,54 +38,37 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B - reference: '' - 12186013-27b2-4e7f-aba7-0c365edcc708: !Template - answer_choices: Yes ||| No - id: 12186013-27b2-4e7f-aba7-0c365edcc708 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences + name: random_choice_1_2_2 reference: '' - 29ab2bf4-22ba-4bf7-a234-c6d8fc4d6fb2: !Template - answer_choices: null - id: 29ab2bf4-22ba-4bf7-a234-c6d8fc4d6fb2 + 2926db8f-3363-4ecb-ad4d-04efaf2828bf: !Template + answer_choices: A ||| B + id: 2926db8f-3363-4ecb-ad4d-04efaf2828bf jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -93,37 +76,70 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: 'A/B choice randomized (choice order: B-A)' reference: '' - 49ed8ab5-946c-47a7-b3fe-d718e07d3cda: !Template - answer_choices: A ||| B - id: 49ed8ab5-946c-47a7-b3fe-d718e07d3cda + 3de3ed27-eaf9-4d17-8853-57cec4ea5256: !Template + answer_choices: Yes ||| No + id: 3de3ed27-eaf9-4d17-8853-57cec4ea5256 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' + reference: '' + 5e459dcd-b32e-4be2-808f-9a33418029aa: !Template + answer_choices: Yes ||| No + id: 5e459dcd-b32e-4be2-808f-9a33418029aa + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' + reference: '' + 80accaec-5e37-4f7d-ac1b-51f7f4fc6d51: !Template + answer_choices: null + id: 80accaec-5e37-4f7d-ac1b-51f7f4fc6d51 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| - {{ answer_choices[0] }} + {% if shuffled_order == 0 %} + + A {% else %} - {{ answer_choices[1] }} + B {% endif %}' metadata: !TemplateMetadata @@ -131,26 +147,26 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized (choice order reversed) + name: random_choice_A_B reference: '' - 86889d8d-79bd-423c-b0d7-746032d59aac: !Template + 986c414b-489b-416c-bdf1-14edbea3bbec: !Template answer_choices: null - id: 86889d8d-79bd-423c-b0d7-746032d59aac + id: 986c414b-489b-416c-bdf1-14edbea3bbec jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -158,11 +174,11 @@ templates: {% if shuffled_order == 0 %} - 1 + A {% else %} - 2 + B {% endif %}' metadata: !TemplateMetadata @@ -170,26 +186,26 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: random_choice_A_B_2 reference: '' - 8e348173-4835-4a2b-b0ba-cdb0711cf101: !Template + 9b737e07-3c3c-4c79-8ec8-0f68abd102b1: !Template answer_choices: null - id: 8e348173-4835-4a2b-b0ba-cdb0711cf101 + id: 9b737e07-3c3c-4c79-8ec8-0f68abd102b1 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -197,11 +213,11 @@ templates: {% if shuffled_order == 0 %} - A + 1 {% else %} - B + 2 {% endif %}' metadata: !TemplateMetadata @@ -209,11 +225,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: random_choice_1_2 reference: '' - 9208808a-ce6a-4777-aae8-974c361b34c3: !Template + a041fb4e-2406-48f9-8058-a8ad1b2629dd: !Template answer_choices: Yes ||| No - id: 9208808a-ce6a-4777-aae8-974c361b34c3 + id: a041fb4e-2406-48f9-8058-a8ad1b2629dd jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -225,15 +241,15 @@ templates: metrics: - Accuracy original_task: false - name: Yes/No binary judgment for good sentences (choice order reversed) + name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' reference: '' - 9acaf85d-e6e8-4d8c-82db-340b70fc8eee: !Template + b1a6eefc-c595-4ce4-8c29-d22303630a9c: !Template answer_choices: Yes ||| No - id: 9acaf85d-e6e8-4d8c-82db-340b70fc8eee + id: b1a6eefc-c595-4ce4-8c29-d22303630a9c jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata @@ -241,11 +257,11 @@ templates: metrics: - Accuracy original_task: false - name: Yes/No binary judgment for bad sentences (choice order reversed) + name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' reference: '' - d1c90a72-23d2-4359-b5de-7328a799d217: !Template + d915a001-fed4-4d8a-8ed3-319d2ecee41e: !Template answer_choices: A ||| B - id: d1c90a72-23d2-4359-b5de-7328a799d217 + id: d915a001-fed4-4d8a-8ed3-319d2ecee41e jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -279,21 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized - reference: '' - d53914cb-8544-48b7-98f2-b89e0f17003d: !Template - answer_choices: Yes ||| No - id: d53914cb-8544-48b7-98f2-b89e0f17003d - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences + name: 'A/B choice randomized (choice order: A-B)' reference: '' diff --git a/promptsource/templates/blimp/determiner_noun_agreement_irregular_1/templates.yaml b/promptsource/templates/blimp/determiner_noun_agreement_irregular_1/templates.yaml index 900a518f1..bb62b3313 100644 --- a/promptsource/templates/blimp/determiner_noun_agreement_irregular_1/templates.yaml +++ b/promptsource/templates/blimp/determiner_noun_agreement_irregular_1/templates.yaml @@ -1,35 +1,36 @@ dataset: blimp subset: determiner_noun_agreement_irregular_1 templates: - 0ebef220-8636-42ab-9cf4-b70067fde695: !Template - answer_choices: A ||| B - id: 0ebef220-8636-42ab-9cf4-b70067fde695 + 1d5e79e9-fc6f-4a0f-9b99-06259c556d8f: !Template + answer_choices: null + id: 1d5e79e9-fc6f-4a0f-9b99-06259c556d8f jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| - {{ answer_choices[0] }} + {% if shuffled_order == 0 %} + + A {% else %} - {{ answer_choices[1] }} + B {% endif %}' metadata: !TemplateMetadata @@ -37,14 +38,46 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized + name: random_choice_A_B + reference: '' + 3107c8b7-a4f3-4083-a33b-32e676b8a7ce: !Template + answer_choices: Yes ||| No + id: 3107c8b7-a4f3-4083-a33b-32e676b8a7ce + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' reference: '' - 15318c27-bda3-4d65-bdd3-38b54b303edb: !Template + 34c18d68-dc3d-442d-a942-06bccfd2ed07: !Template + answer_choices: Yes ||| No + id: 34c18d68-dc3d-442d-a942-06bccfd2ed07 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' + reference: '' + 51bd6ce0-0d94-47bc-b64a-2f15182084e3: !Template answer_choices: null - id: 15318c27-bda3-4d65-bdd3-38b54b303edb + id: 51bd6ce0-0d94-47bc-b64a-2f15182084e3 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -76,49 +109,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 - reference: '' - 19e42a55-4b59-4f78-a989-2ed1b4b0e35a: !Template - answer_choices: A ||| B - id: 19e42a55-4b59-4f78-a989-2ed1b4b0e35a - jinja: '{% set shuffled_order = [0, 1] | random %} - - Read the following two sentences. - - {% if shuffled_order == 0 %} - - Sentence A: {{ sentence_good }} - - Sentence B: {{ sentence_bad }} - - {% else %} - - Sentence A: {{ sentence_bad }} - - Sentence B: {{ sentence_good }} - - {% endif %} - - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} - - {% endif %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: A/B choice randomized (choice order reversed) + name: random_choice_1_2 reference: '' - 3385fbbc-d6db-4859-aea9-c62d80665ec9: !Template + 5388a8c8-ac53-4b26-8253-5878d34ea323: !Template answer_choices: null - id: 3385fbbc-d6db-4859-aea9-c62d80665ec9 + id: 5388a8c8-ac53-4b26-8253-5878d34ea323 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer A or B. @@ -155,13 +150,13 @@ templates: original_task: true name: random_choice_A_B_2 reference: '' - 950220e7-e981-4d1d-b2d0-9e4d3b91ac3b: !Template + 90d6a1bc-fab5-43aa-be78-157d39ca7de1: !Template answer_choices: Yes ||| No - id: 950220e7-e981-4d1d-b2d0-9e4d3b91ac3b + id: 90d6a1bc-fab5-43aa-be78-157d39ca7de1 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata @@ -169,70 +164,75 @@ templates: metrics: - Accuracy original_task: false - name: Yes/No binary judgment for bad sentences + name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' reference: '' - b123bfc6-a416-49a2-8f08-b01c5601d07d: !Template - answer_choices: Yes ||| No - id: b123bfc6-a416-49a2-8f08-b01c5601d07d - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + aa090d09-d35e-4d79-8378-941efa1c96e7: !Template + answer_choices: A ||| B + id: aa090d09-d35e-4d79-8378-941efa1c96e7 + jinja: '{% set shuffled_order = [0, 1] | random %} - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences - reference: '' - b5bac42b-4947-4759-b717-3c364023ac30: !Template - answer_choices: Yes ||| No - id: b5bac42b-4947-4759-b717-3c364023ac30 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + Read the following two sentences. - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences (choice order reversed) + original_task: true + name: 'A/B choice randomized (choice order: B-A)' reference: '' - d6eb4a2f-b5c5-49b1-a6fd-a6538b7c5369: !Template - answer_choices: null - id: d6eb4a2f-b5c5-49b1-a6fd-a6538b7c5369 + bd74eaa5-1999-48ee-9e49-b7638ca17d0e: !Template + answer_choices: A ||| B + id: bd74eaa5-1999-48ee-9e49-b7638ca17d0e jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -240,42 +240,26 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: 'A/B choice randomized (choice order: A-B)' reference: '' - f484a660-c6e4-4238-92bf-13531f1e9453: !Template - answer_choices: Yes ||| No - id: f484a660-c6e4-4238-92bf-13531f1e9453 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences (choice order reversed) - reference: '' - fc625d82-45a2-4d54-a384-72797a29447b: !Template + e4511bf0-e90e-4083-94e4-eeddffce3107: !Template answer_choices: null - id: fc625d82-45a2-4d54-a384-72797a29447b + id: e4511bf0-e90e-4083-94e4-eeddffce3107 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -283,11 +267,11 @@ templates: {% if shuffled_order == 0 %} - A + 1 {% else %} - B + 2 {% endif %}' metadata: !TemplateMetadata @@ -295,5 +279,21 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: random_choice_1_2_2 + reference: '' + f61ddf80-f696-45e8-abe1-583cd2caf963: !Template + answer_choices: Yes ||| No + id: f61ddf80-f696-45e8-abe1-583cd2caf963 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' reference: '' diff --git a/promptsource/templates/blimp/determiner_noun_agreement_irregular_2/templates.yaml b/promptsource/templates/blimp/determiner_noun_agreement_irregular_2/templates.yaml index d9bf71b9b..a6a4c0783 100644 --- a/promptsource/templates/blimp/determiner_noun_agreement_irregular_2/templates.yaml +++ b/promptsource/templates/blimp/determiner_noun_agreement_irregular_2/templates.yaml @@ -1,80 +1,9 @@ dataset: blimp subset: determiner_noun_agreement_irregular_2 templates: - 0228ca33-f383-47ca-a5bb-f0714bfdc73b: !Template - answer_choices: Yes ||| No - id: 0228ca33-f383-47ca-a5bb-f0714bfdc73b - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences (choice order reversed) - reference: '' - 037afad0-dbc5-4cd0-b53e-143ced5ff90e: !Template - answer_choices: Yes ||| No - id: 037afad0-dbc5-4cd0-b53e-143ced5ff90e - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences - reference: '' - 3ad8e304-5acc-4f63-b79c-414467d74339: !Template + 12d29269-013c-428b-bfac-112a9bcfcee0: !Template answer_choices: null - id: 3ad8e304-5acc-4f63-b79c-414467d74339 - jinja: '{% set shuffled_order = [0, 1] | random %} - - Which one of the following sentences is grammatical? Please answer 1 or 2. - - {% if shuffled_order == 0 %} - - 1: {{ sentence_good }} - - 2: {{ sentence_bad }} - - {% else %} - - 1: {{ sentence_bad }} - - 2: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - 1 - - {% else %} - - 2 - - {% endif %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: random_choice_1_2 - reference: '' - 69a4e6fb-6d82-4daa-9681-9c3aa322afdc: !Template - answer_choices: null - id: 69a4e6fb-6d82-4daa-9681-9c3aa322afdc + id: 12d29269-013c-428b-bfac-112a9bcfcee0 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer A or B. @@ -111,9 +40,9 @@ templates: original_task: true name: random_choice_A_B reference: '' - 8a7e61e5-89b2-46bb-9652-04d362b32964: !Template + 1ab8247a-a0cd-4efb-862e-ba4439c6f649: !Template answer_choices: null - id: 8a7e61e5-89b2-46bb-9652-04d362b32964 + id: 1ab8247a-a0cd-4efb-862e-ba4439c6f649 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -150,35 +79,36 @@ templates: original_task: true name: random_choice_1_2_2 reference: '' - ac822a81-03bf-4327-a1d5-3522fa8ca563: !Template - answer_choices: A ||| B - id: ac822a81-03bf-4327-a1d5-3522fa8ca563 + 41da01a4-1c38-4399-9e99-2d9b15811aeb: !Template + answer_choices: null + id: 41da01a4-1c38-4399-9e99-2d9b15811aeb jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| - {{ answer_choices[0] }} + {% if shuffled_order == 0 %} + + 1 {% else %} - {{ answer_choices[1] }} + 2 {% endif %}' metadata: !TemplateMetadata @@ -186,31 +116,15 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized - reference: '' - cd4334e0-2f75-4bf7-bed1-e68b462efd86: !Template - answer_choices: Yes ||| No - id: cd4334e0-2f75-4bf7-bed1-e68b462efd86 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences (choice order reversed) + name: random_choice_1_2 reference: '' - d5a5f664-ea12-4b21-bd1b-05990f8f8578: !Template + 51876660-1c80-4607-a6af-63461ed4172a: !Template answer_choices: Yes ||| No - id: d5a5f664-ea12-4b21-bd1b-05990f8f8578 + id: 51876660-1c80-4607-a6af-63461ed4172a jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata @@ -218,11 +132,11 @@ templates: metrics: - Accuracy original_task: false - name: Yes/No binary judgment for bad sentences + name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' reference: '' - f5d9ced3-ef82-4f19-a700-42f5c5920ce3: !Template + 52c1851e-b0b8-4c09-9e30-9b921b3d0059: !Template answer_choices: null - id: f5d9ced3-ef82-4f19-a700-42f5c5920ce3 + id: 52c1851e-b0b8-4c09-9e30-9b921b3d0059 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer A or B. @@ -259,9 +173,9 @@ templates: original_task: true name: random_choice_A_B_2 reference: '' - ff05613a-97f1-4c60-b9e7-8f7e181d46a9: !Template + 60db8bc5-52f7-4155-b5b4-cc8007c166c6: !Template answer_choices: A ||| B - id: ff05613a-97f1-4c60-b9e7-8f7e181d46a9 + id: 60db8bc5-52f7-4155-b5b4-cc8007c166c6 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -295,5 +209,91 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized (choice order reversed) + name: 'A/B choice randomized (choice order: B-A)' + reference: '' + 697420f0-4e22-45bd-b182-8f7e774a52dd: !Template + answer_choices: A ||| B + id: 697420f0-4e22-45bd-b182-8f7e774a52dd + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice randomized (choice order: A-B)' + reference: '' + a7a653e1-8764-4667-9af5-7f683acc5ad0: !Template + answer_choices: Yes ||| No + id: a7a653e1-8764-4667-9af5-7f683acc5ad0 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' + reference: '' + b42a7a6c-efee-481a-b623-c45f27e60328: !Template + answer_choices: Yes ||| No + id: b42a7a6c-efee-481a-b623-c45f27e60328 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' + reference: '' + e8ca15b8-fbbb-49bd-b12a-80fee5a828c4: !Template + answer_choices: Yes ||| No + id: e8ca15b8-fbbb-49bd-b12a-80fee5a828c4 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' reference: '' diff --git a/promptsource/templates/blimp/determiner_noun_agreement_with_adj_2/templates.yaml b/promptsource/templates/blimp/determiner_noun_agreement_with_adj_2/templates.yaml index 10776bf84..b4ac0e667 100644 --- a/promptsource/templates/blimp/determiner_noun_agreement_with_adj_2/templates.yaml +++ b/promptsource/templates/blimp/determiner_noun_agreement_with_adj_2/templates.yaml @@ -1,9 +1,9 @@ dataset: blimp subset: determiner_noun_agreement_with_adj_2 templates: - 3886eb20-4f0a-45ca-b450-010032068940: !Template + 10b55fcc-3e50-4aac-9ff4-999e20c95892: !Template answer_choices: null - id: 3886eb20-4f0a-45ca-b450-010032068940 + id: 10b55fcc-3e50-4aac-9ff4-999e20c95892 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer A or B. @@ -40,51 +40,52 @@ templates: original_task: true name: random_choice_A_B_2 reference: '' - 478ab9b8-65a0-4fff-9246-56d82dc91d51: !Template + 13ad95ed-9249-464e-ad40-6aba1dc8dfbf: !Template answer_choices: Yes ||| No - id: 478ab9b8-65a0-4fff-9246-56d82dc91d51 + id: 13ad95ed-9249-464e-ad40-6aba1dc8dfbf jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: Yes/No binary judgment for good sentences + name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' reference: '' - 4fc5e43b-0f5b-470f-8cfc-a21ae4aa6d09: !Template - answer_choices: A ||| B - id: 4fc5e43b-0f5b-470f-8cfc-a21ae4aa6d09 + 31efe64e-08b8-45fa-9685-9fedb1d974cf: !Template + answer_choices: null + id: 31efe64e-08b8-45fa-9685-9fedb1d974cf jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| - {{ answer_choices[0] }} + {% if shuffled_order == 0 %} + + 1 {% else %} - {{ answer_choices[1] }} + 2 {% endif %}' metadata: !TemplateMetadata @@ -92,42 +93,26 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized (choice order reversed) - reference: '' - 5d301e88-44c1-4604-800d-96e5824d6e7e: !Template - answer_choices: Yes ||| No - id: 5d301e88-44c1-4604-800d-96e5824d6e7e - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences (choice order reversed) + name: random_choice_1_2 reference: '' - 603299fe-ff70-4158-b31f-6e03828e9de4: !Template + 40cb218a-e034-4292-a4cd-b1ec05f16889: !Template answer_choices: null - id: 603299fe-ff70-4158-b31f-6e03828e9de4 + id: 40cb218a-e034-4292-a4cd-b1ec05f16889 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -135,11 +120,11 @@ templates: {% if shuffled_order == 0 %} - A + 1 {% else %} - B + 2 {% endif %}' metadata: !TemplateMetadata @@ -147,26 +132,26 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: random_choice_1_2_2 reference: '' - a1e1c3a1-495d-40c5-b051-29d0eef99c39: !Template + 79cdbcef-1785-4f19-84f5-91507a1f0c9d: !Template answer_choices: null - id: a1e1c3a1-495d-40c5-b051-29d0eef99c39 + id: 79cdbcef-1785-4f19-84f5-91507a1f0c9d jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -174,11 +159,11 @@ templates: {% if shuffled_order == 0 %} - 1 + A {% else %} - 2 + B {% endif %}' metadata: !TemplateMetadata @@ -186,11 +171,59 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: random_choice_A_B + reference: '' + 7f94d9c9-acb2-4ad1-b78d-3dafa7f7ce8e: !Template + answer_choices: Yes ||| No + id: 7f94d9c9-acb2-4ad1-b78d-3dafa7f7ce8e + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' + reference: '' + 89bd2199-5b38-4269-aeb8-047e12d5970e: !Template + answer_choices: Yes ||| No + id: 89bd2199-5b38-4269-aeb8-047e12d5970e + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' + reference: '' + c2c457a7-2e46-41a9-b2f2-2838855552dc: !Template + answer_choices: Yes ||| No + id: c2c457a7-2e46-41a9-b2f2-2838855552dc + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' reference: '' - a8055f9b-20b2-4497-8d0a-9321036d251b: !Template + cfa06ff8-02bd-4a1b-8737-720010838a62: !Template answer_choices: A ||| B - id: a8055f9b-20b2-4497-8d0a-9321036d251b + id: cfa06ff8-02bd-4a1b-8737-720010838a62 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -224,70 +257,37 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized + name: 'A/B choice randomized (choice order: A-B)' reference: '' - cbd66cab-5519-4b1a-82f0-472145673157: !Template - answer_choices: Yes ||| No - id: cbd66cab-5519-4b1a-82f0-472145673157 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences - reference: '' - d384d2a2-5f92-4e8a-8e87-a4246bd27621: !Template - answer_choices: Yes ||| No - id: d384d2a2-5f92-4e8a-8e87-a4246bd27621 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences (choice order reversed) - reference: '' - fc3a67ec-f95d-4ee7-9ad8-9749b17c2238: !Template - answer_choices: null - id: fc3a67ec-f95d-4ee7-9ad8-9749b17c2238 + d637a459-6f73-4a5d-9b16-cc34ca66ecd3: !Template + answer_choices: A ||| B + id: d637a459-6f73-4a5d-9b16-cc34ca66ecd3 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -295,5 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: 'A/B choice randomized (choice order: B-A)' reference: '' diff --git a/promptsource/templates/blimp/determiner_noun_agreement_with_adj_irregular_1/templates.yaml b/promptsource/templates/blimp/determiner_noun_agreement_with_adj_irregular_1/templates.yaml index b3e61f867..79d3c5809 100644 --- a/promptsource/templates/blimp/determiner_noun_agreement_with_adj_irregular_1/templates.yaml +++ b/promptsource/templates/blimp/determiner_noun_agreement_with_adj_irregular_1/templates.yaml @@ -1,28 +1,28 @@ dataset: blimp subset: determiner_noun_agreement_with_adj_irregular_1 templates: - 148ed02a-e0fa-4a39-a0d4-c2cd2859e711: !Template + 045f504c-76d8-4225-973b-6d835118bd92: !Template answer_choices: Yes ||| No - id: 148ed02a-e0fa-4a39-a0d4-c2cd2859e711 + id: 045f504c-76d8-4225-973b-6d835118bd92 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: Yes/No binary judgment for good sentences + name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' reference: '' - 295d19cb-8ee3-4588-b074-b2b97c4e73ff: !Template + 186beef9-5742-42d8-b75d-c011ad3fc4c5: !Template answer_choices: null - id: 295d19cb-8ee3-4588-b074-b2b97c4e73ff + id: 186beef9-5742-42d8-b75d-c011ad3fc4c5 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -54,11 +54,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: random_choice_1_2_2 reference: '' - 36588004-dc66-4c53-a053-edd3d51130ab: !Template + 6288f326-e84d-4149-83eb-0035ece04ff4: !Template answer_choices: A ||| B - id: 36588004-dc66-4c53-a053-edd3d51130ab + id: 6288f326-e84d-4149-83eb-0035ece04ff4 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -77,7 +77,7 @@ templates: {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order + Which one is a better sentence of English, B or A? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -92,54 +92,37 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized - reference: '' - 62aac670-4c21-4c2d-9855-aada50e253dd: !Template - answer_choices: Yes ||| No - id: 62aac670-4c21-4c2d-9855-aada50e253dd - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences + name: 'A/B choice randomized (choice order: B-A)' reference: '' - 943d13a8-2813-4d7e-8ad8-dab9e6acea25: !Template - answer_choices: null - id: 943d13a8-2813-4d7e-8ad8-dab9e6acea25 + 7a9598d2-8dc3-40fd-a143-033c9cb133c8: !Template + answer_choices: A ||| B + id: 7a9598d2-8dc3-40fd-a143-033c9cb133c8 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -147,11 +130,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: 'A/B choice randomized (choice order: A-B)' reference: '' - a3e6cb80-06d4-4e22-bd58-249436d13d11: !Template + 7c10eaa6-8c76-418d-86de-4ed59c45cdb3: !Template answer_choices: null - id: a3e6cb80-06d4-4e22-bd58-249436d13d11 + id: 7c10eaa6-8c76-418d-86de-4ed59c45cdb3 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer A or B. @@ -188,24 +171,24 @@ templates: original_task: true name: random_choice_A_B reference: '' - b4dc51e3-d5db-4791-a5f3-ff82a08b8b16: !Template + 86a56515-55d5-4ecc-89bd-88b17b885b3d: !Template answer_choices: null - id: b4dc51e3-d5db-4791-a5f3-ff82a08b8b16 + id: 86a56515-55d5-4ecc-89bd-88b17b885b3d jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -213,11 +196,11 @@ templates: {% if shuffled_order == 0 %} - 1 + A {% else %} - 2 + B {% endif %}' metadata: !TemplateMetadata @@ -225,31 +208,31 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: random_choice_A_B_2 reference: '' - b8ac6c90-e8b3-41fe-b2eb-fd01b22c069f: !Template + 95597f80-2588-492c-8972-db0c61d4f105: !Template answer_choices: Yes ||| No - id: b8ac6c90-e8b3-41fe-b2eb-fd01b22c069f + id: 95597f80-2588-492c-8972-db0c61d4f105 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: Yes/No binary judgment for bad sentences (choice order reversed) + name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' reference: '' - f192ada3-cc6e-4249-89fb-e74e2c6c42f1: !Template + 9e886594-52fe-474f-94c3-f6c4dee8f398: !Template answer_choices: Yes ||| No - id: f192ada3-cc6e-4249-89fb-e74e2c6c42f1 + id: 9e886594-52fe-474f-94c3-f6c4dee8f398 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -257,37 +240,38 @@ templates: metrics: - Accuracy original_task: false - name: Yes/No binary judgment for good sentences (choice order reversed) + name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' reference: '' - f8061c99-4d02-4d29-a73b-1bdf50411219: !Template - answer_choices: A ||| B - id: f8061c99-4d02-4d29-a73b-1bdf50411219 + a9207b62-c320-4444-b5f6-75cd7a8a4879: !Template + answer_choices: null + id: a9207b62-c320-4444-b5f6-75cd7a8a4879 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| - {{ answer_choices[0] }} + {% if shuffled_order == 0 %} + + 1 {% else %} - {{ answer_choices[1] }} + 2 {% endif %}' metadata: !TemplateMetadata @@ -295,5 +279,21 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized (choice order reversed) + name: random_choice_1_2 + reference: '' + ff4ec272-1749-4ea5-a522-d9312fb6569d: !Template + answer_choices: Yes ||| No + id: ff4ec272-1749-4ea5-a522-d9312fb6569d + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' reference: '' diff --git a/promptsource/templates/blimp/determiner_noun_agreement_with_adj_irregular_2/templates.yaml b/promptsource/templates/blimp/determiner_noun_agreement_with_adj_irregular_2/templates.yaml index 8e686fab5..1bfb81fce 100644 --- a/promptsource/templates/blimp/determiner_noun_agreement_with_adj_irregular_2/templates.yaml +++ b/promptsource/templates/blimp/determiner_noun_agreement_with_adj_irregular_2/templates.yaml @@ -1,35 +1,36 @@ dataset: blimp subset: determiner_noun_agreement_with_adj_irregular_2 templates: - 1a736052-ab64-4cfa-b233-d5b84cb37076: !Template - answer_choices: A ||| B - id: 1a736052-ab64-4cfa-b233-d5b84cb37076 + 39d35024-fd56-4b59-9890-4cf7ef67c55f: !Template + answer_choices: null + id: 39d35024-fd56-4b59-9890-4cf7ef67c55f jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| - {{ answer_choices[0] }} + {% if shuffled_order == 0 %} + + A {% else %} - {{ answer_choices[1] }} + B {% endif %}' metadata: !TemplateMetadata @@ -37,11 +38,27 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized + name: random_choice_A_B reference: '' - 38508ba7-f900-4193-90c7-dbfaec2cd9c6: !Template + 4bf11505-3a29-4861-ac4c-9bd9871361c7: !Template answer_choices: Yes ||| No - id: 38508ba7-f900-4193-90c7-dbfaec2cd9c6 + id: 4bf11505-3a29-4861-ac4c-9bd9871361c7 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' + reference: '' + 69ba54e3-d0e6-47c3-a1b9-184e7f8321bd: !Template + answer_choices: Yes ||| No + id: 69ba54e3-d0e6-47c3-a1b9-184e7f8321bd jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -53,11 +70,11 @@ templates: metrics: - Accuracy original_task: false - name: Yes/No binary judgment for bad sentences (choice order reversed) + name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' reference: '' - 3e03d8c7-ca52-4644-bb26-6b9b3965582a: !Template + 80dac6de-36f7-4cb8-a538-58e80d6a98e4: !Template answer_choices: Yes ||| No - id: 3e03d8c7-ca52-4644-bb26-6b9b3965582a + id: 80dac6de-36f7-4cb8-a538-58e80d6a98e4 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -69,14 +86,14 @@ templates: metrics: - Accuracy original_task: false - name: Yes/No binary judgment for bad sentences + name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' reference: '' - 432cf8db-1236-44db-9d81-d449e7cf8807: !Template + 89ff4368-1271-4496-abed-312e101fe8b1: !Template answer_choices: null - id: 432cf8db-1236-44db-9d81-d449e7cf8807 + id: 89ff4368-1271-4496-abed-312e101fe8b1 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -108,11 +125,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: random_choice_A_B_2 reference: '' - 443abdcb-d535-4373-8250-377f9041ff74: !Template + a16e57d6-14b1-40d6-a9f7-e24349e46246: !Template answer_choices: null - id: 443abdcb-d535-4373-8250-377f9041ff74 + id: a16e57d6-14b1-40d6-a9f7-e24349e46246 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -149,24 +166,24 @@ templates: original_task: true name: random_choice_1_2_2 reference: '' - bb790191-ae53-4a76-a9b2-3f5e6592aade: !Template + b5b0072c-fbef-4cd0-91b7-e52c2db5b561: !Template answer_choices: null - id: bb790191-ae53-4a76-a9b2-3f5e6592aade + id: b5b0072c-fbef-4cd0-91b7-e52c2db5b561 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -174,11 +191,11 @@ templates: {% if shuffled_order == 0 %} - A + 1 {% else %} - B + 2 {% endif %}' metadata: !TemplateMetadata @@ -186,27 +203,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 - reference: '' - bd329ec8-ed8c-4a6d-844e-d185bf9c63e6: !Template - answer_choices: Yes ||| No - id: bd329ec8-ed8c-4a6d-844e-d185bf9c63e6 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences + name: random_choice_1_2 reference: '' - be89a4c0-3de5-4066-b1a6-c70b809c1b84: !Template + c14756c3-1e8e-4982-a3a3-20495198880f: !Template answer_choices: A ||| B - id: be89a4c0-3de5-4066-b1a6-c70b809c1b84 + id: c14756c3-1e8e-4982-a3a3-20495198880f jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -225,7 +226,7 @@ templates: {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order + Which one is a better sentence of English, A or B? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -240,54 +241,37 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized (choice order reversed) - reference: '' - ca9cf663-9ced-44d2-8a09-f4bdc3c34f60: !Template - answer_choices: Yes ||| No - id: ca9cf663-9ced-44d2-8a09-f4bdc3c34f60 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences (choice order reversed) + name: 'A/B choice randomized (choice order: A-B)' reference: '' - dc194ba6-f216-4bb9-948d-38d52f1d00d4: !Template - answer_choices: null - id: dc194ba6-f216-4bb9-948d-38d52f1d00d4 + f1d757a3-d31a-4c19-a23f-73a6d9cfdbb0: !Template + answer_choices: A ||| B + id: f1d757a3-d31a-4c19-a23f-73a6d9cfdbb0 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -295,5 +279,21 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: 'A/B choice randomized (choice order: B-A)' + reference: '' + fee5c349-3bf7-4bc9-b190-fdf2c0c27c3d: !Template + answer_choices: Yes ||| No + id: fee5c349-3bf7-4bc9-b190-fdf2c0c27c3d + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' reference: '' diff --git a/promptsource/templates/blimp/determiner_noun_agreement_with_adjective_1/templates.yaml b/promptsource/templates/blimp/determiner_noun_agreement_with_adjective_1/templates.yaml index cb11c4a08..4cf86557d 100644 --- a/promptsource/templates/blimp/determiner_noun_agreement_with_adjective_1/templates.yaml +++ b/promptsource/templates/blimp/determiner_noun_agreement_with_adjective_1/templates.yaml @@ -1,9 +1,80 @@ dataset: blimp subset: determiner_noun_agreement_with_adjective_1 templates: - 14dc5768-494f-4c60-ab44-136f5d2e4b18: !Template + 02004ede-c0aa-4210-b96c-ad62c10ed818: !Template + answer_choices: Yes ||| No + id: 02004ede-c0aa-4210-b96c-ad62c10ed818 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' + reference: '' + 11231ce2-c610-4fdc-96bd-8545bec4b174: !Template + answer_choices: null + id: 11231ce2-c610-4fdc-96bd-8545bec4b174 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2_2 + reference: '' + 152f85f8-555b-4065-83e7-a11395d6d463: !Template + answer_choices: Yes ||| No + id: 152f85f8-555b-4065-83e7-a11395d6d463 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' + reference: '' + 1c768f5c-9028-4f5e-a5e7-9a7ff83b548c: !Template answer_choices: null - id: 14dc5768-494f-4c60-ab44-136f5d2e4b18 + id: 1c768f5c-9028-4f5e-a5e7-9a7ff83b548c jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer 1 or 2. @@ -40,12 +111,12 @@ templates: original_task: true name: random_choice_1_2 reference: '' - 2288835c-890b-497c-98e0-0a6e441e487a: !Template + 51ffb0c5-488e-40a8-8048-4436b0962548: !Template answer_choices: null - id: 2288835c-890b-497c-98e0-0a6e441e487a + id: 51ffb0c5-488e-40a8-8048-4436b0962548 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -77,27 +148,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 - reference: '' - 8ca00c13-f9e1-4f21-a87f-0d901596b15b: !Template - answer_choices: Yes ||| No - id: 8ca00c13-f9e1-4f21-a87f-0d901596b15b - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences + name: random_choice_A_B reference: '' - 9e312167-48ae-48ea-9e35-91dd20cc1b99: !Template + 94f6be35-8f67-4298-abcd-ccaaca33ce1f: !Template answer_choices: Yes ||| No - id: 9e312167-48ae-48ea-9e35-91dd20cc1b99 + id: 94f6be35-8f67-4298-abcd-ccaaca33ce1f jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -109,27 +164,11 @@ templates: metrics: - Accuracy original_task: false - name: Yes/No binary judgment for good sentences (choice order reversed) - reference: '' - 9eb5c3b0-82de-46c9-a44d-1c62a045876b: !Template - answer_choices: Yes ||| No - id: 9eb5c3b0-82de-46c9-a44d-1c62a045876b - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences (choice order reversed) + name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' reference: '' - a0b1f044-dc67-4f43-9d55-1f738ce93ce5: !Template + 9cc90e49-8d9b-486d-9ec3-15829173017e: !Template answer_choices: A ||| B - id: a0b1f044-dc67-4f43-9d55-1f738ce93ce5 + id: 9cc90e49-8d9b-486d-9ec3-15829173017e jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -148,7 +187,7 @@ templates: {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order + Which one is a better sentence of English, B or A? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -163,11 +202,27 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized + name: 'A/B choice randomized (choice order: B-A)' reference: '' - b305581a-7188-4073-90b8-2f11d9ede93a: !Template + b5ec67eb-6356-474a-a54d-a94f2717983d: !Template + answer_choices: Yes ||| No + id: b5ec67eb-6356-474a-a54d-a94f2717983d + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' + reference: '' + d676ab08-308d-48f2-aab1-50211261475e: !Template answer_choices: A ||| B - id: b305581a-7188-4073-90b8-2f11d9ede93a + id: d676ab08-308d-48f2-aab1-50211261475e jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -186,7 +241,7 @@ templates: {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order + Which one is a better sentence of English, A or B? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -201,14 +256,14 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized (choice order reversed) + name: 'A/B choice randomized (choice order: A-B)' reference: '' - cc1dbecb-6c8c-43a3-9508-53b8e37a3d6c: !Template + da246a25-3001-421f-86d6-e07b11281f06: !Template answer_choices: null - id: cc1dbecb-6c8c-43a3-9508-53b8e37a3d6c + id: da246a25-3001-421f-86d6-e07b11281f06 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -240,60 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B - reference: '' - d441f455-c16a-4daa-98c5-a831bdda0d76: !Template - answer_choices: null - id: d441f455-c16a-4daa-98c5-a831bdda0d76 - jinja: '{% set shuffled_order = [0, 1] | random %} - - Between the two sentences, which one is grammatical? Please answer 1 or 2. - - {% if shuffled_order == 0 %} - - 1: {{ sentence_good }} - - 2: {{ sentence_bad }} - - {% else %} - - 1: {{ sentence_bad }} - - 2: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - 1 - - {% else %} - - 2 - - {% endif %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: random_choice_1_2_2 - reference: '' - e9831865-c53f-4784-84f1-bd92e5920d27: !Template - answer_choices: Yes ||| No - id: e9831865-c53f-4784-84f1-bd92e5920d27 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences + name: random_choice_A_B_2 reference: '' diff --git a/promptsource/templates/blimp/distractor_agreement_relational_noun/templates.yaml b/promptsource/templates/blimp/distractor_agreement_relational_noun/templates.yaml index 90a06ebc2..88aa0090d 100644 --- a/promptsource/templates/blimp/distractor_agreement_relational_noun/templates.yaml +++ b/promptsource/templates/blimp/distractor_agreement_relational_noun/templates.yaml @@ -1,51 +1,36 @@ dataset: blimp subset: distractor_agreement_relational_noun templates: - 4c188be7-ba6a-4555-8b3b-329a9586aa58: !Template - answer_choices: Yes ||| No - id: 4c188be7-ba6a-4555-8b3b-329a9586aa58 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences - reference: '' - 52f83bdc-0af7-407f-8eb2-30f0b82b0861: !Template - answer_choices: A ||| B - id: 52f83bdc-0af7-407f-8eb2-30f0b82b0861 + 0093546a-ad1b-4211-99d7-32298a95a6c8: !Template + answer_choices: null + id: 0093546a-ad1b-4211-99d7-32298a95a6c8 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| - {{ answer_choices[0] }} + {% if shuffled_order == 0 %} + + A {% else %} - {{ answer_choices[1] }} + B {% endif %}' metadata: !TemplateMetadata @@ -53,26 +38,26 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized (choice order reversed) + name: random_choice_A_B_2 reference: '' - 79712317-8ceb-4dda-98b5-2ede1f561f1d: !Template + 556132b2-dc63-4d0a-9979-bd36ec5adefb: !Template answer_choices: null - id: 79712317-8ceb-4dda-98b5-2ede1f561f1d + id: 556132b2-dc63-4d0a-9979-bd36ec5adefb jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -80,11 +65,11 @@ templates: {% if shuffled_order == 0 %} - A + 1 {% else %} - B + 2 {% endif %}' metadata: !TemplateMetadata @@ -92,47 +77,69 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: random_choice_1_2 reference: '' - 8b503be3-6928-430c-b9ee-b53d985320a8: !Template - answer_choices: Yes ||| No - id: 8b503be3-6928-430c-b9ee-b53d985320a8 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + 7d643f07-11f5-4948-8971-d2870478cb9a: !Template + answer_choices: A ||| B + id: 7d643f07-11f5-4948-8971-d2870478cb9a + jinja: '{% set shuffled_order = [0, 1] | random %} - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences + original_task: true + name: 'A/B choice randomized (choice order: B-A)' reference: '' - 9da6e7fe-1b21-411a-a8ba-1967de0c55d1: !Template + 7e62f966-8cfa-4e6b-b57b-39a6650ad434: !Template answer_choices: Yes ||| No - id: 9da6e7fe-1b21-411a-a8ba-1967de0c55d1 + id: 7e62f966-8cfa-4e6b-b57b-39a6650ad434 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: Yes/No binary judgment for bad sentences (choice order reversed) + name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' reference: '' - b92c7d16-50fc-4274-85fd-1c3a7a938fba: !Template + 7f41d5b1-dbd2-4a6d-9383-6cf3e8340228: !Template answer_choices: Yes ||| No - id: b92c7d16-50fc-4274-85fd-1c3a7a938fba + id: 7f41d5b1-dbd2-4a6d-9383-6cf3e8340228 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -140,26 +147,26 @@ templates: metrics: - Accuracy original_task: false - name: Yes/No binary judgment for good sentences (choice order reversed) + name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' reference: '' - c47853cd-67a8-4d0c-af68-68d09334e843: !Template + 9bcba8ef-215d-4c53-8855-c7139ce89f87: !Template answer_choices: null - id: c47853cd-67a8-4d0c-af68-68d09334e843 + id: 9bcba8ef-215d-4c53-8855-c7139ce89f87 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -167,11 +174,11 @@ templates: {% if shuffled_order == 0 %} - A + 1 {% else %} - B + 2 {% endif %}' metadata: !TemplateMetadata @@ -179,26 +186,26 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: random_choice_1_2_2 reference: '' - c61c531b-ad4e-49ca-a3d1-f066d4e8df15: !Template + a50f9477-e887-47ab-86c3-25bbe13bde5f: !Template answer_choices: null - id: c61c531b-ad4e-49ca-a3d1-f066d4e8df15 + id: a50f9477-e887-47ab-86c3-25bbe13bde5f jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -206,11 +213,11 @@ templates: {% if shuffled_order == 0 %} - 1 + A {% else %} - 2 + B {% endif %}' metadata: !TemplateMetadata @@ -218,11 +225,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: random_choice_A_B reference: '' - cd9e9e21-7e96-4d47-8f5f-87183286edf1: !Template + bf30aa91-05b9-4339-8608-d68470a1dec1: !Template answer_choices: A ||| B - id: cd9e9e21-7e96-4d47-8f5f-87183286edf1 + id: bf30aa91-05b9-4339-8608-d68470a1dec1 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -256,44 +263,37 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized + name: 'A/B choice randomized (choice order: A-B)' reference: '' - ec5b64f9-1b5c-45ae-a7f3-ce42f3ca12c3: !Template - answer_choices: null - id: ec5b64f9-1b5c-45ae-a7f3-ce42f3ca12c3 - jinja: '{% set shuffled_order = [0, 1] | random %} - - Which one of the following sentences is grammatical? Please answer 1 or 2. - - {% if shuffled_order == 0 %} - - 1: {{ sentence_good }} - - 2: {{ sentence_bad }} - - {% else %} - - 1: {{ sentence_bad }} - - 2: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - 1 - - {% else %} + bfcde61d-b884-44c2-8e17-6cd8f98af970: !Template + answer_choices: Yes ||| No + id: bfcde61d-b884-44c2-8e17-6cd8f98af970 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - 2 + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' + reference: '' + ddb15e2e-07ec-401a-bd45-494f2f182103: !Template + answer_choices: Yes ||| No + id: ddb15e2e-07ec-401a-bd45-494f2f182103 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - {% endif %}' + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: random_choice_1_2 + original_task: false + name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' reference: '' diff --git a/promptsource/templates/blimp/distractor_agreement_relative_clause/templates.yaml b/promptsource/templates/blimp/distractor_agreement_relative_clause/templates.yaml index 6a40bb1f8..fdcd29a91 100644 --- a/promptsource/templates/blimp/distractor_agreement_relative_clause/templates.yaml +++ b/promptsource/templates/blimp/distractor_agreement_relative_clause/templates.yaml @@ -1,36 +1,35 @@ dataset: blimp subset: distractor_agreement_relative_clause templates: - 1ef86170-8434-41a9-840a-1bd16b6fd17c: !Template - answer_choices: null - id: 1ef86170-8434-41a9-840a-1bd16b6fd17c + 04107f2b-4ab2-4403-a3fd-ed945870209c: !Template + answer_choices: A ||| B + id: 04107f2b-4ab2-4403-a3fd-ed945870209c jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -38,11 +37,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: 'A/B choice randomized (choice order: B-A)' reference: '' - 34b4f384-755a-4cb1-b0ed-83143381d446: !Template + 0f9a8483-1559-41ee-ab1b-db7ae5255d2b: !Template answer_choices: Yes ||| No - id: 34b4f384-755a-4cb1-b0ed-83143381d446 + id: 0f9a8483-1559-41ee-ab1b-db7ae5255d2b jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -54,30 +53,53 @@ templates: metrics: - Accuracy original_task: false - name: Yes/No binary judgment for good sentences + name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' reference: '' - 3d9f64a4-36e7-4521-bfc9-9d51d238abfc: !Template - answer_choices: Yes ||| No - id: 3d9f64a4-36e7-4521-bfc9-9d51d238abfc - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + 24371907-ca39-4ee3-957b-a6391cb2a50a: !Template + answer_choices: null + id: 24371907-ca39-4ee3-957b-a6391cb2a50a + jinja: '{% set shuffled_order = [0, 1] | random %} - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences (choice order reversed) + original_task: true + name: random_choice_1_2 reference: '' - 91da17d3-ef43-4a68-9de0-729aca73b6cc: !Template + 2f019572-dfb6-4c6f-adfa-3cd5764e90e2: !Template answer_choices: null - id: 91da17d3-ef43-4a68-9de0-729aca73b6cc + id: 2f019572-dfb6-4c6f-adfa-3cd5764e90e2 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -109,53 +131,38 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 - reference: '' - a203a23d-1584-4fc9-8b30-6628050e64f6: !Template - answer_choices: Yes ||| No - id: a203a23d-1584-4fc9-8b30-6628050e64f6 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences + name: random_choice_A_B reference: '' - c42987e1-8f08-4e3a-b0e1-d27560a7570c: !Template - answer_choices: A ||| B - id: c42987e1-8f08-4e3a-b0e1-d27560a7570c + 604ea06e-a8b5-47d0-9fcf-43ac57a5ad9d: !Template + answer_choices: null + id: 604ea06e-a8b5-47d0-9fcf-43ac57a5ad9d jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| - {{ answer_choices[0] }} + {% if shuffled_order == 0 %} + + A {% else %} - {{ answer_choices[1] }} + B {% endif %}' metadata: !TemplateMetadata @@ -163,42 +170,42 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized (choice order reversed) + name: random_choice_A_B_2 reference: '' - cf654a40-7389-4115-a78e-929687e209ca: !Template + ca808663-bc8e-453b-81a0-bcc48e0612e1: !Template answer_choices: Yes ||| No - id: cf654a40-7389-4115-a78e-929687e209ca + id: ca808663-bc8e-453b-81a0-bcc48e0612e1 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: Yes/No binary judgment for good sentences (choice order reversed) + name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' reference: '' - e93eb5d0-f7de-4380-9ec7-4b51b09f80b7: !Template + cb995c35-e892-44ba-86bd-51c5e5be9cae: !Template answer_choices: null - id: e93eb5d0-f7de-4380-9ec7-4b51b09f80b7 + id: cb995c35-e892-44ba-86bd-51c5e5be9cae jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -206,11 +213,11 @@ templates: {% if shuffled_order == 0 %} - A + 1 {% else %} - B + 2 {% endif %}' metadata: !TemplateMetadata @@ -218,11 +225,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: random_choice_1_2_2 reference: '' - f2c44dce-ea0b-4736-b602-7edf37bafc7a: !Template + e06e1a13-8350-4b80-8ef2-7b7f16ccf8e2: !Template answer_choices: A ||| B - id: f2c44dce-ea0b-4736-b602-7edf37bafc7a + id: e06e1a13-8350-4b80-8ef2-7b7f16ccf8e2 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -256,44 +263,37 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized + name: 'A/B choice randomized (choice order: A-B)' reference: '' - f5301a1c-4aea-42d2-a36a-1dc04c89733a: !Template - answer_choices: null - id: f5301a1c-4aea-42d2-a36a-1dc04c89733a - jinja: '{% set shuffled_order = [0, 1] | random %} - - Which one of the following sentences is grammatical? Please answer 1 or 2. - - {% if shuffled_order == 0 %} - - 1: {{ sentence_good }} - - 2: {{ sentence_bad }} - - {% else %} - - 1: {{ sentence_bad }} - - 2: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - 1 - - {% else %} + e9f2ca2b-59f3-4150-b6fc-54d27b3fdb91: !Template + answer_choices: Yes ||| No + id: e9f2ca2b-59f3-4150-b6fc-54d27b3fdb91 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - 2 + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' + reference: '' + f0376181-8db5-46c2-ada3-d7333545f1f4: !Template + answer_choices: Yes ||| No + id: f0376181-8db5-46c2-ada3-d7333545f1f4 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - {% endif %}' + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: random_choice_1_2 + original_task: false + name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' reference: '' diff --git a/promptsource/templates/blimp/drop_argument/templates.yaml b/promptsource/templates/blimp/drop_argument/templates.yaml index 7a8c7d4a7..6470f6706 100644 --- a/promptsource/templates/blimp/drop_argument/templates.yaml +++ b/promptsource/templates/blimp/drop_argument/templates.yaml @@ -1,41 +1,9 @@ dataset: blimp subset: drop_argument templates: - 0bf1ba17-165d-40f0-9b9a-05d585d8809d: !Template - answer_choices: Yes ||| No - id: 0bf1ba17-165d-40f0-9b9a-05d585d8809d - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences - reference: '' - 1870d493-7858-454d-8bf2-5991584b3b5b: !Template - answer_choices: Yes ||| No - id: 1870d493-7858-454d-8bf2-5991584b3b5b - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences (choice order reversed) - reference: '' - 787990bf-9ad9-4855-91e8-58de425ed0dc: !Template + 0a28fdc6-e122-4146-99f9-e2dbe41129d0: !Template answer_choices: null - id: 787990bf-9ad9-4855-91e8-58de425ed0dc + id: 0a28fdc6-e122-4146-99f9-e2dbe41129d0 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer 1 or 2. @@ -72,40 +40,24 @@ templates: original_task: true name: random_choice_1_2 reference: '' - 83e5275e-eec3-473a-a661-86de738ce309: !Template - answer_choices: Yes ||| No - id: 83e5275e-eec3-473a-a661-86de738ce309 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences (choice order reversed) - reference: '' - a40019ff-ec9a-4bf9-9939-6843907680ea: !Template + 0cf5e865-8718-4c83-a3cc-ceaade91a933: !Template answer_choices: null - id: a40019ff-ec9a-4bf9-9939-6843907680ea + id: 0cf5e865-8718-4c83-a3cc-ceaade91a933 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -113,11 +65,11 @@ templates: {% if shuffled_order == 0 %} - 1 + A {% else %} - 2 + B {% endif %}' metadata: !TemplateMetadata @@ -125,26 +77,58 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: random_choice_A_B + reference: '' + 117b47a4-0b41-4d0e-8dc1-2c2eb76befda: !Template + answer_choices: Yes ||| No + id: 117b47a4-0b41-4d0e-8dc1-2c2eb76befda + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' + reference: '' + 27988d5f-9575-460e-a572-5dc150e2ac32: !Template + answer_choices: Yes ||| No + id: 27988d5f-9575-460e-a572-5dc150e2ac32 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' reference: '' - a8f2fb21-8838-4405-a871-4de7fd62aa47: !Template + 27d930bd-417f-4468-a2f1-3e700832c3ba: !Template answer_choices: null - id: a8f2fb21-8838-4405-a871-4de7fd62aa47 + id: 27d930bd-417f-4468-a2f1-3e700832c3ba jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -152,11 +136,11 @@ templates: {% if shuffled_order == 0 %} - A + 1 {% else %} - B + 2 {% endif %}' metadata: !TemplateMetadata @@ -164,11 +148,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: random_choice_1_2_2 reference: '' - d02f9f83-bfcd-41fa-b4d8-97781ce9561b: !Template + 350e7dfd-e545-418b-bb5d-70a4a6c7a310: !Template answer_choices: A ||| B - id: d02f9f83-bfcd-41fa-b4d8-97781ce9561b + id: 350e7dfd-e545-418b-bb5d-70a4a6c7a310 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -202,30 +186,14 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized (choice order reversed) - reference: '' - d981d4f8-78c9-4891-b54c-49f28a4142d6: !Template - answer_choices: Yes ||| No - id: d981d4f8-78c9-4891-b54c-49f28a4142d6 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences + name: 'A/B choice randomized (choice order: B-A)' reference: '' - dd8d9afc-4217-4960-babe-e281776fe11b: !Template + 3b4b4b58-052c-4505-b7e7-a2e17c17506a: !Template answer_choices: null - id: dd8d9afc-4217-4960-babe-e281776fe11b + id: 3b4b4b58-052c-4505-b7e7-a2e17c17506a jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -257,11 +225,27 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: random_choice_A_B_2 reference: '' - df229dea-02cd-4d69-85d7-386de829b690: !Template + 5a3b706b-62d8-4ea4-9a41-da7a60b4d231: !Template + answer_choices: Yes ||| No + id: 5a3b706b-62d8-4ea4-9a41-da7a60b4d231 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' + reference: '' + a2f7052f-4615-4323-a468-210db95f3951: !Template answer_choices: A ||| B - id: df229dea-02cd-4d69-85d7-386de829b690 + id: a2f7052f-4615-4323-a468-210db95f3951 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -295,5 +279,21 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized + name: 'A/B choice randomized (choice order: A-B)' + reference: '' + f94a694a-ee62-412b-ad6f-071ed15e1204: !Template + answer_choices: Yes ||| No + id: f94a694a-ee62-412b-ad6f-071ed15e1204 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' reference: '' diff --git a/promptsource/templates/blimp/ellipsis_n_bar_1/templates.yaml b/promptsource/templates/blimp/ellipsis_n_bar_1/templates.yaml index a29c42aa1..0aad747fe 100644 --- a/promptsource/templates/blimp/ellipsis_n_bar_1/templates.yaml +++ b/promptsource/templates/blimp/ellipsis_n_bar_1/templates.yaml @@ -1,12 +1,28 @@ dataset: blimp subset: ellipsis_n_bar_1 templates: - 01a44c1f-0795-46f2-bcbe-fe0c4d913640: !Template + 3752e43c-6907-4cb0-a5be-251704c4227d: !Template + answer_choices: Yes ||| No + id: 3752e43c-6907-4cb0-a5be-251704c4227d + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' + reference: '' + 522aa603-fad1-4272-84f2-8069a4d5f0f5: !Template answer_choices: null - id: 01a44c1f-0795-46f2-bcbe-fe0c4d913640 + id: 522aa603-fad1-4272-84f2-8069a4d5f0f5 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -38,11 +54,50 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: random_choice_A_B_2 reference: '' - 3eac83e5-11ca-46d4-9b0e-0516b6088971: !Template + 7655f660-5815-4e41-9f55-a3a3400ff80c: !Template + answer_choices: null + id: 7655f660-5815-4e41-9f55-a3a3400ff80c + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2 + reference: '' + 964bcb7a-5364-44ec-8fa8-eb21308387c6: !Template answer_choices: Yes ||| No - id: 3eac83e5-11ca-46d4-9b0e-0516b6088971 + id: 964bcb7a-5364-44ec-8fa8-eb21308387c6 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -54,14 +109,30 @@ templates: metrics: - Accuracy original_task: false - name: Yes/No binary judgment for good sentences + name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' reference: '' - 8f6d4252-e867-4e73-884e-e68ff6ea3b07: !Template + 97bfbcd8-13a8-4be9-b2d6-32a2d1c9fecf: !Template + answer_choices: Yes ||| No + id: 97bfbcd8-13a8-4be9-b2d6-32a2d1c9fecf + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' + reference: '' + b55558f4-73c9-4d70-a80c-b16e8c459963: !Template answer_choices: null - id: 8f6d4252-e867-4e73-884e-e68ff6ea3b07 + id: b55558f4-73c9-4d70-a80c-b16e8c459963 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -93,11 +164,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: random_choice_A_B reference: '' - 9c2181f4-f828-4486-9bc6-66b474a5b8b1: !Template + bbb01915-634c-4728-8a06-573cab8996f5: !Template answer_choices: A ||| B - id: 9c2181f4-f828-4486-9bc6-66b474a5b8b1 + id: bbb01915-634c-4728-8a06-573cab8996f5 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -116,7 +187,7 @@ templates: {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order + Which one is a better sentence of English, B or A? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -131,82 +202,27 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized - reference: '' - a257c15a-e3a3-4362-a3b4-44623a93e608: !Template - answer_choices: Yes ||| No - id: a257c15a-e3a3-4362-a3b4-44623a93e608 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences (choice order reversed) + name: 'A/B choice randomized (choice order: B-A)' reference: '' - ce12d33f-ee98-49b8-825b-0f0b98ddde80: !Template + d7c98f47-a090-4462-bc92-59994ab98636: !Template answer_choices: Yes ||| No - id: ce12d33f-ee98-49b8-825b-0f0b98ddde80 + id: d7c98f47-a090-4462-bc92-59994ab98636 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: Yes/No binary judgment for bad sentences + name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' reference: '' - da36c020-4ca0-40be-aa54-96628a9e1903: !Template + dd32551a-04e6-43e9-b6ce-0f28343ad36b: !Template answer_choices: null - id: da36c020-4ca0-40be-aa54-96628a9e1903 - jinja: '{% set shuffled_order = [0, 1] | random %} - - Which one of the following sentences is grammatical? Please answer 1 or 2. - - {% if shuffled_order == 0 %} - - 1: {{ sentence_good }} - - 2: {{ sentence_bad }} - - {% else %} - - 1: {{ sentence_bad }} - - 2: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - 1 - - {% else %} - - 2 - - {% endif %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: random_choice_1_2 - reference: '' - e07ae199-198b-4bae-9405-2a306c0cebe1: !Template - answer_choices: null - id: e07ae199-198b-4bae-9405-2a306c0cebe1 + id: dd32551a-04e6-43e9-b6ce-0f28343ad36b jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -243,25 +259,9 @@ templates: original_task: true name: random_choice_1_2_2 reference: '' - e8f7cb79-ec4c-466d-a332-295727f12985: !Template - answer_choices: Yes ||| No - id: e8f7cb79-ec4c-466d-a332-295727f12985 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences (choice order reversed) - reference: '' - efc5e0a3-5b64-46da-afb6-1f71208300e7: !Template + f0d98fcc-bb1c-47ad-8501-8059becb674d: !Template answer_choices: A ||| B - id: efc5e0a3-5b64-46da-afb6-1f71208300e7 + id: f0d98fcc-bb1c-47ad-8501-8059becb674d jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -280,7 +280,7 @@ templates: {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order + Which one is a better sentence of English, A or B? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -295,5 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized (choice order reversed) + name: 'A/B choice randomized (choice order: A-B)' reference: '' diff --git a/promptsource/templates/blimp/ellipsis_n_bar_2/templates.yaml b/promptsource/templates/blimp/ellipsis_n_bar_2/templates.yaml index 3ca9f67cc..ad9aa33c2 100644 --- a/promptsource/templates/blimp/ellipsis_n_bar_2/templates.yaml +++ b/promptsource/templates/blimp/ellipsis_n_bar_2/templates.yaml @@ -1,13 +1,13 @@ dataset: blimp subset: ellipsis_n_bar_2 templates: - 057469f7-603d-44f3-ae1e-1bc6e8629fdd: !Template + 0b780cde-1f1f-4dba-8a71-b7668d7da967: !Template answer_choices: Yes ||| No - id: 057469f7-603d-44f3-ae1e-1bc6e8629fdd + id: 0b780cde-1f1f-4dba-8a71-b7668d7da967 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -15,14 +15,14 @@ templates: metrics: - Accuracy original_task: false - name: Yes/No binary judgment for good sentences (choice order reversed) + name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' reference: '' - 2ee1824b-0c3c-43c3-b401-6ec17d7b539f: !Template + 1e37fcb2-4f57-4fd2-b529-617b0a43b09c: !Template answer_choices: null - id: 2ee1824b-0c3c-43c3-b401-6ec17d7b539f + id: 1e37fcb2-4f57-4fd2-b529-617b0a43b09c jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -54,27 +54,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 - reference: '' - 31fd3e98-fe31-4faf-a773-ff89327a6be9: !Template - answer_choices: Yes ||| No - id: 31fd3e98-fe31-4faf-a773-ff89327a6be9 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences (choice order reversed) + name: random_choice_1_2_2 reference: '' - 3ce21da7-d059-465d-99b7-202e9ab53141: !Template + 378f0de8-ea2b-40c0-b33d-16ad3b1bf1d9: !Template answer_choices: A ||| B - id: 3ce21da7-d059-465d-99b7-202e9ab53141 + id: 378f0de8-ea2b-40c0-b33d-16ad3b1bf1d9 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -93,7 +77,7 @@ templates: {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order + Which one is a better sentence of English, B or A? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -108,26 +92,42 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized + name: 'A/B choice randomized (choice order: B-A)' + reference: '' + 80371c00-36d5-4c85-935a-cccbb98aea43: !Template + answer_choices: Yes ||| No + id: 80371c00-36d5-4c85-935a-cccbb98aea43 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' reference: '' - 61cdb2a7-8710-43de-a1a9-4d537edda041: !Template + 89972adf-1163-408e-9412-cf89c8845282: !Template answer_choices: null - id: 61cdb2a7-8710-43de-a1a9-4d537edda041 + id: 89972adf-1163-408e-9412-cf89c8845282 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -135,11 +135,11 @@ templates: {% if shuffled_order == 0 %} - A + 1 {% else %} - B + 2 {% endif %}' metadata: !TemplateMetadata @@ -147,37 +147,38 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: random_choice_1_2 reference: '' - 7613236f-2c75-4bfd-8f24-e9a85e21d123: !Template - answer_choices: A ||| B - id: 7613236f-2c75-4bfd-8f24-e9a85e21d123 + b9f576ab-362d-4876-9f0a-02d9392ff2f6: !Template + answer_choices: null + id: b9f576ab-362d-4876-9f0a-02d9392ff2f6 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| - {{ answer_choices[0] }} + {% if shuffled_order == 0 %} + + A {% else %} - {{ answer_choices[1] }} + B {% endif %}' metadata: !TemplateMetadata @@ -185,38 +186,53 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized (choice order reversed) + name: random_choice_A_B reference: '' - 93ae78e3-c813-4286-b3e7-fea762ccc90b: !Template - answer_choices: null - id: 93ae78e3-c813-4286-b3e7-fea762ccc90b + bd6b8408-b8e4-4f6d-9b67-74062d0cb696: !Template + answer_choices: Yes ||| No + id: bd6b8408-b8e4-4f6d-9b67-74062d0cb696 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' + reference: '' + c492684e-aa70-488a-9098-2f06ef52cf1d: !Template + answer_choices: A ||| B + id: c492684e-aa70-488a-9098-2f06ef52cf1d jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -224,11 +240,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: 'A/B choice randomized (choice order: A-B)' reference: '' - 9f359e11-ac9a-4132-bfec-627657e99d98: !Template + d58eae09-f150-43a2-9f81-17c25ec82d30: !Template answer_choices: Yes ||| No - id: 9f359e11-ac9a-4132-bfec-627657e99d98 + id: d58eae09-f150-43a2-9f81-17c25ec82d30 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -240,30 +256,14 @@ templates: metrics: - Accuracy original_task: false - name: Yes/No binary judgment for bad sentences + name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' reference: '' - a37850c7-1d04-4644-8400-cdcc2fa78ada: !Template - answer_choices: Yes ||| No - id: a37850c7-1d04-4644-8400-cdcc2fa78ada - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences - reference: '' - f8fd5129-ef43-4859-9e4a-6b52f3f98368: !Template + dbc37147-56c2-4898-b632-66db234e6d21: !Template answer_choices: null - id: f8fd5129-ef43-4859-9e4a-6b52f3f98368 + id: dbc37147-56c2-4898-b632-66db234e6d21 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -295,5 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: random_choice_A_B_2 reference: '' diff --git a/promptsource/templates/blimp/existential_there_object_raising/templates.yaml b/promptsource/templates/blimp/existential_there_object_raising/templates.yaml index 6c9c9f7b6..5f28f6548 100644 --- a/promptsource/templates/blimp/existential_there_object_raising/templates.yaml +++ b/promptsource/templates/blimp/existential_there_object_raising/templates.yaml @@ -1,24 +1,40 @@ dataset: blimp subset: existential_there_object_raising templates: - 09d49336-b67a-4fd8-a1f2-227cfa11f348: !Template + 28a55ee2-d8f6-4fc3-8008-e60150559377: !Template + answer_choices: Yes ||| No + id: 28a55ee2-d8f6-4fc3-8008-e60150559377 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' + reference: '' + 6921d555-378b-4629-8d75-a11bad991c5a: !Template answer_choices: null - id: 09d49336-b67a-4fd8-a1f2-227cfa11f348 + id: 6921d555-378b-4629-8d75-a11bad991c5a jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -26,11 +42,11 @@ templates: {% if shuffled_order == 0 %} - A + 1 {% else %} - B + 2 {% endif %}' metadata: !TemplateMetadata @@ -38,11 +54,27 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: random_choice_1_2 reference: '' - 363802c7-c284-44d2-ad3a-5f82a3f3aca2: !Template + 776cdf8b-ee71-4b59-9873-a0d3a07ae77d: !Template + answer_choices: Yes ||| No + id: 776cdf8b-ee71-4b59-9873-a0d3a07ae77d + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' + reference: '' + 8edb1a66-2e13-47f4-92c5-804ce8e947a2: !Template answer_choices: A ||| B - id: 363802c7-c284-44d2-ad3a-5f82a3f3aca2 + id: 8edb1a66-2e13-47f4-92c5-804ce8e947a2 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -76,37 +108,54 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized (choice order reversed) + name: 'A/B choice randomized (choice order: B-A)' reference: '' - 5691f73d-2d67-49f2-ae26-fe93a173d89b: !Template - answer_choices: A ||| B - id: 5691f73d-2d67-49f2-ae26-fe93a173d89b + a0cad0f5-22f7-43ae-a3ab-b52c8b842d28: !Template + answer_choices: Yes ||| No + id: a0cad0f5-22f7-43ae-a3ab-b52c8b842d28 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' + reference: '' + be07094b-ed75-4118-b2da-024077dedd26: !Template + answer_choices: null + id: be07094b-ed75-4118-b2da-024077dedd26 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| - {{ answer_choices[0] }} + {% if shuffled_order == 0 %} + + A {% else %} - {{ answer_choices[1] }} + B {% endif %}' metadata: !TemplateMetadata @@ -114,11 +163,11 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized + name: random_choice_A_B_2 reference: '' - 647bbd76-ae39-44d8-9240-6e4a4ce69783: !Template + cb9a8767-7450-4074-8037-aa054f24db81: !Template answer_choices: Yes ||| No - id: 647bbd76-ae39-44d8-9240-6e4a4ce69783 + id: cb9a8767-7450-4074-8037-aa054f24db81 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -130,26 +179,26 @@ templates: metrics: - Accuracy original_task: false - name: Yes/No binary judgment for bad sentences + name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' reference: '' - 8665f45a-d651-495e-b947-70795e79952e: !Template + e21618f9-8973-4236-91f6-c88ad17fe846: !Template answer_choices: null - id: 8665f45a-d651-495e-b947-70795e79952e + id: e21618f9-8973-4236-91f6-c88ad17fe846 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -157,11 +206,11 @@ templates: {% if shuffled_order == 0 %} - 1 + A {% else %} - 2 + B {% endif %}' metadata: !TemplateMetadata @@ -169,27 +218,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 - reference: '' - a2ffe96e-ca1c-47f0-93d5-d84cca7f3bb7: !Template - answer_choices: Yes ||| No - id: a2ffe96e-ca1c-47f0-93d5-d84cca7f3bb7 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences (choice order reversed) + name: random_choice_A_B reference: '' - af85dede-9050-430c-9993-5bb4a90c94b8: !Template + e72a0571-fbee-47d5-bad0-18c039550f99: !Template answer_choices: null - id: af85dede-9050-430c-9993-5bb4a90c94b8 + id: e72a0571-fbee-47d5-bad0-18c039550f99 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -226,68 +259,35 @@ templates: original_task: true name: random_choice_1_2_2 reference: '' - b37c08ad-edac-4599-bd7f-38adc42a5fd4: !Template - answer_choices: Yes ||| No - id: b37c08ad-edac-4599-bd7f-38adc42a5fd4 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences (choice order reversed) - reference: '' - cf7da09c-ad33-4adc-b210-beca6b849501: !Template - answer_choices: Yes ||| No - id: cf7da09c-ad33-4adc-b210-beca6b849501 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences - reference: '' - d2f71332-89e9-47b9-89a6-32b651accfec: !Template - answer_choices: null - id: d2f71332-89e9-47b9-89a6-32b651accfec + fc31a373-2721-4623-a0ae-f91eb8c2ef36: !Template + answer_choices: A ||| B + id: fc31a373-2721-4623-a0ae-f91eb8c2ef36 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -295,5 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: 'A/B choice randomized (choice order: A-B)' reference: '' diff --git a/promptsource/templates/blimp/existential_there_quantifiers_1/templates.yaml b/promptsource/templates/blimp/existential_there_quantifiers_1/templates.yaml index 338829700..134921c89 100644 --- a/promptsource/templates/blimp/existential_there_quantifiers_1/templates.yaml +++ b/promptsource/templates/blimp/existential_there_quantifiers_1/templates.yaml @@ -1,12 +1,12 @@ dataset: blimp subset: existential_there_quantifiers_1 templates: - 0ea04c16-ffaa-456a-8829-db4bb63e6d2b: !Template + 0233dd43-9281-4398-a134-1083fde228aa: !Template answer_choices: null - id: 0ea04c16-ffaa-456a-8829-db4bb63e6d2b + id: 0233dd43-9281-4398-a134-1083fde228aa jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -38,27 +38,50 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: random_choice_A_B_2 reference: '' - 2c4d2d8d-d44a-4117-a1d5-06df8e6b19f5: !Template - answer_choices: Yes ||| No - id: 2c4d2d8d-d44a-4117-a1d5-06df8e6b19f5 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + 0e144c21-4ab8-40d1-8ac5-32a4bf0c8f49: !Template + answer_choices: null + id: 0e144c21-4ab8-40d1-8ac5-32a4bf0c8f49 + jinja: '{% set shuffled_order = [0, 1] | random %} - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences + original_task: true + name: random_choice_1_2 reference: '' - 3bccd33b-c9b6-4a6e-886c-764c23387bda: !Template + 3b60d1d4-eccf-46be-a1c3-2d848175ad62: !Template answer_choices: A ||| B - id: 3bccd33b-c9b6-4a6e-886c-764c23387bda + id: 3b60d1d4-eccf-46be-a1c3-2d848175ad62 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -77,7 +100,7 @@ templates: {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order + Which one is a better sentence of English, B or A? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -92,42 +115,26 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized - reference: '' - 4fe1f1c8-fadd-4281-980d-bf2e7643e2b5: !Template - answer_choices: Yes ||| No - id: 4fe1f1c8-fadd-4281-980d-bf2e7643e2b5 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences + name: 'A/B choice randomized (choice order: B-A)' reference: '' - 580ef5e1-c4a5-41f5-a76d-c063aed4ab00: !Template + 51ebc961-3d22-4723-bb9b-39a930c43615: !Template answer_choices: null - id: 580ef5e1-c4a5-41f5-a76d-c063aed4ab00 + id: 51ebc961-3d22-4723-bb9b-39a930c43615 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -135,11 +142,11 @@ templates: {% if shuffled_order == 0 %} - 1 + A {% else %} - 2 + B {% endif %}' metadata: !TemplateMetadata @@ -147,42 +154,42 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: random_choice_A_B reference: '' - 617cf5f3-4ae5-461b-af29-cb2a124cba1e: !Template + 55469408-4881-4c2a-a4b7-a0c23d31c52f: !Template answer_choices: Yes ||| No - id: 617cf5f3-4ae5-461b-af29-cb2a124cba1e + id: 55469408-4881-4c2a-a4b7-a0c23d31c52f jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: Yes/No binary judgment for bad sentences (choice order reversed) + name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' reference: '' - 89eb62ff-6431-4bab-974c-344e03ddd222: !Template + 559cca5d-4e00-400b-a161-ced8cd373d5d: !Template answer_choices: null - id: 89eb62ff-6431-4bab-974c-344e03ddd222 + id: 559cca5d-4e00-400b-a161-ced8cd373d5d jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -190,11 +197,11 @@ templates: {% if shuffled_order == 0 %} - A + 1 {% else %} - B + 2 {% endif %}' metadata: !TemplateMetadata @@ -202,11 +209,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: random_choice_1_2_2 reference: '' - 93993a21-914d-4a24-80dd-a76fdb1e0884: !Template + 78895df4-c60c-4be2-ba06-eb0f5ddbdb4a: !Template answer_choices: A ||| B - id: 93993a21-914d-4a24-80dd-a76fdb1e0884 + id: 78895df4-c60c-4be2-ba06-eb0f5ddbdb4a jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -225,7 +232,7 @@ templates: {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order + Which one is a better sentence of English, A or B? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -240,15 +247,15 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized (choice order reversed) + name: 'A/B choice randomized (choice order: A-B)' reference: '' - 9af98d9b-17f6-4d8c-8567-2d7eb64037cd: !Template + 79d8ce45-cbe4-486a-ab84-7426e4f30790: !Template answer_choices: Yes ||| No - id: 9af98d9b-17f6-4d8c-8567-2d7eb64037cd + id: 79d8ce45-cbe4-486a-ab84-7426e4f30790 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -256,44 +263,37 @@ templates: metrics: - Accuracy original_task: false - name: Yes/No binary judgment for good sentences (choice order reversed) + name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' reference: '' - a6e4de47-8868-48a6-beb9-72ea62ca9286: !Template - answer_choices: null - id: a6e4de47-8868-48a6-beb9-72ea62ca9286 - jinja: '{% set shuffled_order = [0, 1] | random %} - - Between the two sentences, which one is grammatical? Please answer 1 or 2. - - {% if shuffled_order == 0 %} - - 1: {{ sentence_good }} - - 2: {{ sentence_bad }} - - {% else %} - - 1: {{ sentence_bad }} - - 2: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - 1 - - {% else %} + 9e82e459-6af7-4c62-b382-eeb7c88ed5b3: !Template + answer_choices: Yes ||| No + id: 9e82e459-6af7-4c62-b382-eeb7c88ed5b3 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - 2 + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' + reference: '' + c93a3c8e-35ca-4a86-bcb2-dc9c19ee423e: !Template + answer_choices: Yes ||| No + id: c93a3c8e-35ca-4a86-bcb2-dc9c19ee423e + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - {% endif %}' + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: random_choice_1_2_2 + original_task: false + name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' reference: '' diff --git a/promptsource/templates/blimp/existential_there_quantifiers_2/templates.yaml b/promptsource/templates/blimp/existential_there_quantifiers_2/templates.yaml index be905507e..546ecc206 100644 --- a/promptsource/templates/blimp/existential_there_quantifiers_2/templates.yaml +++ b/promptsource/templates/blimp/existential_there_quantifiers_2/templates.yaml @@ -1,24 +1,24 @@ dataset: blimp subset: existential_there_quantifiers_2 templates: - 1f4dc210-9f43-417b-b9d5-3d0c320513f9: !Template + 228f312c-752d-4813-87d4-d20f50bf1e65: !Template answer_choices: null - id: 1f4dc210-9f43-417b-b9d5-3d0c320513f9 + id: 228f312c-752d-4813-87d4-d20f50bf1e65 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -26,11 +26,11 @@ templates: {% if shuffled_order == 0 %} - 1 + A {% else %} - 2 + B {% endif %}' metadata: !TemplateMetadata @@ -38,26 +38,42 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: random_choice_A_B_2 reference: '' - 5aa1a0d9-9e17-4cf9-8e2d-d3b6eaf28d6b: !Template + 3c1d7194-b4da-4123-9546-e7a94a1c6a07: !Template + answer_choices: Yes ||| No + id: 3c1d7194-b4da-4123-9546-e7a94a1c6a07 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' + reference: '' + 5ea32d10-921f-4358-b1de-f93ee3a325f5: !Template answer_choices: null - id: 5aa1a0d9-9e17-4cf9-8e2d-d3b6eaf28d6b + id: 5ea32d10-921f-4358-b1de-f93ee3a325f5 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -65,11 +81,11 @@ templates: {% if shuffled_order == 0 %} - A + 1 {% else %} - B + 2 {% endif %}' metadata: !TemplateMetadata @@ -77,11 +93,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: random_choice_1_2 reference: '' - 69ab9d17-de35-456e-a109-62a2ebb018db: !Template + 629052b1-e99b-490f-ace0-a369011de80e: !Template answer_choices: A ||| B - id: 69ab9d17-de35-456e-a109-62a2ebb018db + id: 629052b1-e99b-490f-ace0-a369011de80e jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -115,37 +131,70 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized (choice order reversed) + name: 'A/B choice randomized (choice order: B-A)' reference: '' - 76d6c700-febc-4dad-91f3-536a74b4996d: !Template - answer_choices: A ||| B - id: 76d6c700-febc-4dad-91f3-536a74b4996d + 69b7033a-0bfb-4412-b846-401710062776: !Template + answer_choices: Yes ||| No + id: 69b7033a-0bfb-4412-b846-401710062776 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' + reference: '' + 8c52b91f-2bd4-443e-b721-2b539b6b153c: !Template + answer_choices: Yes ||| No + id: 8c52b91f-2bd4-443e-b721-2b539b6b153c + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' + reference: '' + ca69fdaa-db5b-4cb6-8d3e-2db4b5933e18: !Template + answer_choices: null + id: ca69fdaa-db5b-4cb6-8d3e-2db4b5933e18 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| - {{ answer_choices[0] }} + {% if shuffled_order == 0 %} + + A {% else %} - {{ answer_choices[1] }} + B {% endif %}' metadata: !TemplateMetadata @@ -153,30 +202,30 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized + name: random_choice_A_B reference: '' - 885625f9-0946-4b4a-b024-781259ff2540: !Template + dc29e12a-b8fa-4791-a90e-faaaca8471c4: !Template answer_choices: Yes ||| No - id: 885625f9-0946-4b4a-b024-781259ff2540 + id: dc29e12a-b8fa-4791-a90e-faaaca8471c4 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: Yes/No binary judgment for good sentences (choice order reversed) + name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' reference: '' - 9b3d0cc7-0a21-4267-b924-7b9fcb54e7dd: !Template + ddb8419e-c454-4a03-8fa3-853d65aa3565: !Template answer_choices: null - id: 9b3d0cc7-0a21-4267-b924-7b9fcb54e7dd + id: ddb8419e-c454-4a03-8fa3-853d65aa3565 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -208,54 +257,37 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 - reference: '' - b4fffc65-d9d8-41c8-a1cf-ff08de5eccc5: !Template - answer_choices: Yes ||| No - id: b4fffc65-d9d8-41c8-a1cf-ff08de5eccc5 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences + name: random_choice_1_2_2 reference: '' - bf7e7a58-f63e-4435-b544-6ce48e58e015: !Template - answer_choices: null - id: bf7e7a58-f63e-4435-b544-6ce48e58e015 + e424dc78-4dc2-429d-ade4-5e18a2cd41b1: !Template + answer_choices: A ||| B + id: e424dc78-4dc2-429d-ade4-5e18a2cd41b1 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -263,37 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 - reference: '' - c4abd7d4-057a-41de-9c6e-7bf05e6fe197: !Template - answer_choices: Yes ||| No - id: c4abd7d4-057a-41de-9c6e-7bf05e6fe197 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences (choice order reversed) - reference: '' - d6896ade-c253-4291-b946-ed3b815c531e: !Template - answer_choices: Yes ||| No - id: d6896ade-c253-4291-b946-ed3b815c531e - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences + name: 'A/B choice randomized (choice order: A-B)' reference: '' diff --git a/promptsource/templates/blimp/existential_there_subject_raising/templates.yaml b/promptsource/templates/blimp/existential_there_subject_raising/templates.yaml index 3fda7566a..8d14628d0 100644 --- a/promptsource/templates/blimp/existential_there_subject_raising/templates.yaml +++ b/promptsource/templates/blimp/existential_there_subject_raising/templates.yaml @@ -1,12 +1,28 @@ dataset: blimp subset: existential_there_subject_raising templates: - 01383138-9dc9-4aec-9d30-29a3bc45396a: !Template + 00796a33-5b42-4651-b901-36e08ab9dea6: !Template + answer_choices: Yes ||| No + id: 00796a33-5b42-4651-b901-36e08ab9dea6 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' + reference: '' + 16b36ab4-e933-433c-9c9e-19147c95bebe: !Template answer_choices: null - id: 01383138-9dc9-4aec-9d30-29a3bc45396a + id: 16b36ab4-e933-433c-9c9e-19147c95bebe jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -38,38 +54,37 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: random_choice_A_B_2 reference: '' - 333bed79-11a0-48f8-8c4c-d920082ce7c8: !Template - answer_choices: null - id: 333bed79-11a0-48f8-8c4c-d920082ce7c8 + 61b49b7b-e775-4ad4-ae18-d19f89bb792f: !Template + answer_choices: A ||| B + id: 61b49b7b-e775-4ad4-ae18-d19f89bb792f jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -77,15 +92,15 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: 'A/B choice randomized (choice order: B-A)' reference: '' - 6311612b-fccf-4e5b-bc63-f674f02134ce: !Template + 81129215-698a-449d-a299-3f38c06fb0ba: !Template answer_choices: Yes ||| No - id: 6311612b-fccf-4e5b-bc63-f674f02134ce + id: 81129215-698a-449d-a299-3f38c06fb0ba jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -93,14 +108,52 @@ templates: metrics: - Accuracy original_task: false - name: Yes/No binary judgment for good sentences + name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' + reference: '' + 875fac89-3cd6-46ce-b544-71e5bf2bde4f: !Template + answer_choices: A ||| B + id: 875fac89-3cd6-46ce-b544-71e5bf2bde4f + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice randomized (choice order: A-B)' reference: '' - 76c481ad-4017-48cd-9b51-9285c4c485d9: !Template + 8f9b0b81-cc49-43c5-918e-afd2df6a0914: !Template answer_choices: null - id: 76c481ad-4017-48cd-9b51-9285c4c485d9 + id: 8f9b0b81-cc49-43c5-918e-afd2df6a0914 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -132,27 +185,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 - reference: '' - 7d67cd77-1744-4ab2-bd17-5e45a4ebe7ab: !Template - answer_choices: Yes ||| No - id: 7d67cd77-1744-4ab2-bd17-5e45a4ebe7ab - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences (choice order reversed) + name: random_choice_A_B reference: '' - 838ab848-4ec8-4dda-bcbc-ed8018ae1dce: !Template + c753e480-c114-4dee-9b03-c5351b09a47f: !Template answer_choices: null - id: 838ab848-4ec8-4dda-bcbc-ed8018ae1dce + id: c753e480-c114-4dee-9b03-c5351b09a47f jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer 1 or 2. @@ -189,13 +226,13 @@ templates: original_task: true name: random_choice_1_2 reference: '' - 9757090f-3401-40d3-921f-b5a686b23423: !Template + d6f43663-86b6-44ed-976f-95added0fb4e: !Template answer_choices: Yes ||| No - id: 9757090f-3401-40d3-921f-b5a686b23423 + id: d6f43663-86b6-44ed-976f-95added0fb4e jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata @@ -203,91 +240,54 @@ templates: metrics: - Accuracy original_task: false - name: Yes/No binary judgment for bad sentences (choice order reversed) + name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' reference: '' - a046e56c-aec4-4d20-abc9-bd519221ba13: !Template + e5f3b1c7-ac29-4791-9eb6-aa98612b8f9a: !Template answer_choices: Yes ||| No - id: a046e56c-aec4-4d20-abc9-bd519221ba13 + id: e5f3b1c7-ac29-4791-9eb6-aa98612b8f9a jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: Yes/No binary judgment for bad sentences + name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' reference: '' - b5f72555-cbba-4156-b4dd-4c071ddabd34: !Template - answer_choices: A ||| B - id: b5f72555-cbba-4156-b4dd-4c071ddabd34 + e75c847d-96b2-4618-b616-fbbe88a2af0e: !Template + answer_choices: null + id: e75c847d-96b2-4618-b616-fbbe88a2af0e jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} - - {% endif %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: A/B choice randomized (choice order reversed) - reference: '' - d3055b67-c20e-4354-8f2c-bcb7da50775a: !Template - answer_choices: A ||| B - id: d3055b67-c20e-4354-8f2c-bcb7da50775a - jinja: '{% set shuffled_order = [0, 1] | random %} - - Read the following two sentences. + ||| {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} - - Sentence B: {{ sentence_bad }} - - {% else %} - - Sentence A: {{ sentence_bad }} - - Sentence B: {{ sentence_good }} - - {% endif %} - - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} - - {{ answer_choices[0] }} + 1 {% else %} - {{ answer_choices[1] }} + 2 {% endif %}' metadata: !TemplateMetadata @@ -295,5 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized + name: random_choice_1_2_2 reference: '' diff --git a/promptsource/templates/blimp/expletive_it_object_raising/templates.yaml b/promptsource/templates/blimp/expletive_it_object_raising/templates.yaml index a7c3486d5..06e8eaac5 100644 --- a/promptsource/templates/blimp/expletive_it_object_raising/templates.yaml +++ b/promptsource/templates/blimp/expletive_it_object_raising/templates.yaml @@ -1,25 +1,25 @@ dataset: blimp subset: expletive_it_object_raising templates: - 6dbec4b3-8a72-4f82-b6b5-1b5dbd5af858: !Template + 07225df9-d8a8-4715-b7e0-9a38fa5f527b: !Template answer_choices: Yes ||| No - id: 6dbec4b3-8a72-4f82-b6b5-1b5dbd5af858 + id: 07225df9-d8a8-4715-b7e0-9a38fa5f527b jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: Yes/No binary judgment for good sentences + name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' reference: '' - 6eebdcd0-e41a-4370-96bc-53af32d41ba8: !Template + 339a7a6d-d2c8-45c9-bb1c-a089a084f099: !Template answer_choices: Yes ||| No - id: 6eebdcd0-e41a-4370-96bc-53af32d41ba8 + id: 339a7a6d-d2c8-45c9-bb1c-a089a084f099 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -31,37 +31,38 @@ templates: metrics: - Accuracy original_task: false - name: Yes/No binary judgment for bad sentences (choice order reversed) + name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' reference: '' - 874c6fbb-1977-45db-8791-5b1173efc30a: !Template - answer_choices: A ||| B - id: 874c6fbb-1977-45db-8791-5b1173efc30a + 341b5119-123a-4101-b75e-ab0fee57f534: !Template + answer_choices: null + id: 341b5119-123a-4101-b75e-ab0fee57f534 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| - {{ answer_choices[0] }} + {% if shuffled_order == 0 %} + + 1 {% else %} - {{ answer_choices[1] }} + 2 {% endif %}' metadata: !TemplateMetadata @@ -69,38 +70,37 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized + name: random_choice_1_2_2 reference: '' - 8f833560-5017-49dc-adee-53f39f9f9290: !Template - answer_choices: null - id: 8f833560-5017-49dc-adee-53f39f9f9290 + 58d8a96f-1c6a-48e4-8545-716f60211a5e: !Template + answer_choices: A ||| B + id: 58d8a96f-1c6a-48e4-8545-716f60211a5e jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -108,11 +108,43 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: 'A/B choice randomized (choice order: A-B)' + reference: '' + 83410312-22b9-4ff2-a3af-387752a104e6: !Template + answer_choices: Yes ||| No + id: 83410312-22b9-4ff2-a3af-387752a104e6 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' + reference: '' + 853fd6b5-7bf4-40bf-99f2-8401cff9d5c1: !Template + answer_choices: Yes ||| No + id: 853fd6b5-7bf4-40bf-99f2-8401cff9d5c1 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' reference: '' - abf4136a-e2c5-4630-9e3b-b9aa9f92e4e0: !Template + 9bd3eea0-08e6-42d0-8088-6589e67d5578: !Template answer_choices: A ||| B - id: abf4136a-e2c5-4630-9e3b-b9aa9f92e4e0 + id: 9bd3eea0-08e6-42d0-8088-6589e67d5578 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -146,14 +178,14 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized (choice order reversed) + name: 'A/B choice randomized (choice order: B-A)' reference: '' - b785ae87-1ca0-4128-a2d6-ead44ef50db9: !Template + b91f51aa-e97e-4d13-86e5-d965aeb8783a: !Template answer_choices: null - id: b785ae87-1ca0-4128-a2d6-ead44ef50db9 + id: b91f51aa-e97e-4d13-86e5-d965aeb8783a jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -185,14 +217,14 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: random_choice_A_B reference: '' - c49fca45-1fa3-45f7-9fe6-2ea4106fa2f0: !Template + eafec0bf-8db2-47d4-bc13-4ec30d2b1d37: !Template answer_choices: null - id: c49fca45-1fa3-45f7-9fe6-2ea4106fa2f0 + id: eafec0bf-8db2-47d4-bc13-4ec30d2b1d37 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -224,14 +256,14 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: random_choice_1_2 reference: '' - c991d156-56dd-4adf-b9f7-bbc68020636b: !Template + fc121e5a-867e-4c74-8f2b-9419a8a8c984: !Template answer_choices: null - id: c991d156-56dd-4adf-b9f7-bbc68020636b + id: fc121e5a-867e-4c74-8f2b-9419a8a8c984 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -263,37 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B - reference: '' - db008d32-4ee6-49be-a81c-513d7b79800e: !Template - answer_choices: Yes ||| No - id: db008d32-4ee6-49be-a81c-513d7b79800e - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences (choice order reversed) - reference: '' - e42fd39f-cb34-45c7-bb8f-37ac6d1d3a8e: !Template - answer_choices: Yes ||| No - id: e42fd39f-cb34-45c7-bb8f-37ac6d1d3a8e - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences + name: random_choice_A_B_2 reference: '' diff --git a/promptsource/templates/blimp/inchoative/templates.yaml b/promptsource/templates/blimp/inchoative/templates.yaml index db09aff5d..da70a52b1 100644 --- a/promptsource/templates/blimp/inchoative/templates.yaml +++ b/promptsource/templates/blimp/inchoative/templates.yaml @@ -1,9 +1,9 @@ dataset: blimp subset: inchoative templates: - 17f72074-130f-4ab4-92a3-c05f3624150d: !Template + 18fd76ff-1ae0-4e76-b89a-87c6821dbc24: !Template answer_choices: Yes ||| No - id: 17f72074-130f-4ab4-92a3-c05f3624150d + id: 18fd76ff-1ae0-4e76-b89a-87c6821dbc24 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -15,11 +15,11 @@ templates: metrics: - Accuracy original_task: false - name: Yes/No binary judgment for bad sentences (choice order reversed) + name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' reference: '' - 1dac225a-ac99-4cce-8385-60a79a1e58a8: !Template + 4c38c931-44e4-452f-a798-37050ee40bb2: !Template answer_choices: A ||| B - id: 1dac225a-ac99-4cce-8385-60a79a1e58a8 + id: 4c38c931-44e4-452f-a798-37050ee40bb2 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -38,7 +38,7 @@ templates: {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order + Which one is a better sentence of English, B or A? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -53,53 +53,54 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized + name: 'A/B choice randomized (choice order: B-A)' reference: '' - 1dba4dae-8844-4c1f-8978-77f97eeb9473: !Template + 5c5c1067-0b10-4d55-9d8d-ea15a6eba76b: !Template answer_choices: Yes ||| No - id: 1dba4dae-8844-4c1f-8978-77f97eeb9473 + id: 5c5c1067-0b10-4d55-9d8d-ea15a6eba76b jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: Yes/No binary judgment for good sentences (choice order reversed) + name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' reference: '' - 52dd1d46-1d61-4af1-97fb-f576fdd74c7a: !Template - answer_choices: A ||| B - id: 52dd1d46-1d61-4af1-97fb-f576fdd74c7a + 70cf6acb-c3bf-4fb9-a5fc-6ab341fe8d3f: !Template + answer_choices: null + id: 70cf6acb-c3bf-4fb9-a5fc-6ab341fe8d3f jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| - {{ answer_choices[0] }} + {% if shuffled_order == 0 %} + + A {% else %} - {{ answer_choices[1] }} + B {% endif %}' metadata: !TemplateMetadata @@ -107,54 +108,53 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized (choice order reversed) + name: random_choice_A_B_2 reference: '' - 7f5828c1-e35d-4995-a558-114eb4c44948: !Template + 7ea9cc59-e8da-40f5-bd1a-8177df6d7785: !Template answer_choices: Yes ||| No - id: 7f5828c1-e35d-4995-a558-114eb4c44948 + id: 7ea9cc59-e8da-40f5-bd1a-8177df6d7785 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: Yes/No binary judgment for bad sentences + name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' reference: '' - 92e7c397-7543-4938-9df3-cff08f6bbdc3: !Template - answer_choices: null - id: 92e7c397-7543-4938-9df3-cff08f6bbdc3 + 9be06dc8-6052-4faf-b6e3-2648ca0c6830: !Template + answer_choices: A ||| B + id: 9be06dc8-6052-4faf-b6e3-2648ca0c6830 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -162,14 +162,14 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: 'A/B choice randomized (choice order: A-B)' reference: '' - 98a12cf3-be23-41dc-b746-cf75b558da5e: !Template + a20e3b81-3480-4899-a9b2-0c795c620dde: !Template answer_choices: null - id: 98a12cf3-be23-41dc-b746-cf75b558da5e + id: a20e3b81-3480-4899-a9b2-0c795c620dde jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -201,26 +201,42 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: random_choice_1_2 + reference: '' + ac31a784-8c88-4abc-b219-0cae1bfccacf: !Template + answer_choices: Yes ||| No + id: ac31a784-8c88-4abc-b219-0cae1bfccacf + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' reference: '' - b008366e-2194-48a7-948d-d9ddcb9d67f8: !Template + cf2c7766-6953-4b83-b5ac-55c20b957b33: !Template answer_choices: null - id: b008366e-2194-48a7-948d-d9ddcb9d67f8 + id: cf2c7766-6953-4b83-b5ac-55c20b957b33 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -228,11 +244,11 @@ templates: {% if shuffled_order == 0 %} - A + 1 {% else %} - B + 2 {% endif %}' metadata: !TemplateMetadata @@ -240,30 +256,14 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B - reference: '' - f858363c-0a4b-4700-8c81-12f437359b3c: !Template - answer_choices: Yes ||| No - id: f858363c-0a4b-4700-8c81-12f437359b3c - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences + name: random_choice_1_2_2 reference: '' - ffcae245-ec53-4d85-a781-4323f78db347: !Template + f5efb6e6-b12a-4dc0-8832-503d519afb8e: !Template answer_choices: null - id: ffcae245-ec53-4d85-a781-4323f78db347 + id: f5efb6e6-b12a-4dc0-8832-503d519afb8e jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -295,5 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: random_choice_A_B reference: '' diff --git a/promptsource/templates/blimp/intransitive/templates.yaml b/promptsource/templates/blimp/intransitive/templates.yaml index bf1843a0f..c36e3c558 100644 --- a/promptsource/templates/blimp/intransitive/templates.yaml +++ b/promptsource/templates/blimp/intransitive/templates.yaml @@ -1,9 +1,9 @@ dataset: blimp subset: intransitive templates: - 1f1303e7-bc53-4836-b6ac-758e3eb84cdd: !Template + 12123396-d2ac-4b76-9ccf-f7cf87d8dba5: !Template answer_choices: A ||| B - id: 1f1303e7-bc53-4836-b6ac-758e3eb84cdd + id: 12123396-d2ac-4b76-9ccf-f7cf87d8dba5 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -37,26 +37,26 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized (choice order reversed) + name: 'A/B choice randomized (choice order: B-A)' reference: '' - 2b7ee9fc-ab5e-4b32-84ab-cb3418f76bf0: !Template + 855a5c06-316b-469d-9183-ce64ae57a5db: !Template answer_choices: null - id: 2b7ee9fc-ab5e-4b32-84ab-cb3418f76bf0 + id: 855a5c06-316b-469d-9183-ce64ae57a5db jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -64,11 +64,11 @@ templates: {% if shuffled_order == 0 %} - 1 + A {% else %} - 2 + B {% endif %}' metadata: !TemplateMetadata @@ -76,11 +76,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: random_choice_A_B reference: '' - 2f4d7fbd-94d2-4737-a3f1-41a92754e5ef: !Template + 87b1169a-409c-491b-9ff9-f5b737fe4e6c: !Template answer_choices: Yes ||| No - id: 2f4d7fbd-94d2-4737-a3f1-41a92754e5ef + id: 87b1169a-409c-491b-9ff9-f5b737fe4e6c jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -92,26 +92,26 @@ templates: metrics: - Accuracy original_task: false - name: Yes/No binary judgment for good sentences (choice order reversed) + name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' reference: '' - 3c6db6ab-16bf-420c-bfba-adae7e0c4718: !Template + a79100c2-73a8-4b09-bd05-d5974aeff0b6: !Template answer_choices: null - id: 3c6db6ab-16bf-420c-bfba-adae7e0c4718 + id: a79100c2-73a8-4b09-bd05-d5974aeff0b6 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -119,11 +119,11 @@ templates: {% if shuffled_order == 0 %} - 1 + A {% else %} - 2 + B {% endif %}' metadata: !TemplateMetadata @@ -131,43 +131,49 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: random_choice_A_B_2 reference: '' - 49604859-53a6-4da5-ad45-80b5e2333885: !Template - answer_choices: Yes ||| No - id: 49604859-53a6-4da5-ad45-80b5e2333885 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + b2cb7019-a27b-4705-8c70-72d9a7d10646: !Template + answer_choices: A ||| B + id: b2cb7019-a27b-4705-8c70-72d9a7d10646 + jinja: '{% set shuffled_order = [0, 1] | random %} - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences - reference: '' - 5487d46b-af72-4b63-9995-a8da08c151c7: !Template - answer_choices: Yes ||| No - id: 5487d46b-af72-4b63-9995-a8da08c151c7 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + Read the following two sentences. - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences (choice order reversed) + original_task: true + name: 'A/B choice randomized (choice order: A-B)' reference: '' - 83d2f463-9170-4253-9d67-bf55836360f9: !Template + b968a4ab-ed23-48cd-9293-2c826072e91b: !Template answer_choices: Yes ||| No - id: 83d2f463-9170-4253-9d67-bf55836360f9 + id: b968a4ab-ed23-48cd-9293-2c826072e91b jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -179,26 +185,26 @@ templates: metrics: - Accuracy original_task: false - name: Yes/No binary judgment for good sentences + name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' reference: '' - aa5703bf-8cf8-4143-ae1b-b8597e6e8f3a: !Template + c6459efe-bece-4aba-86e8-76fc06de16fb: !Template answer_choices: null - id: aa5703bf-8cf8-4143-ae1b-b8597e6e8f3a + id: c6459efe-bece-4aba-86e8-76fc06de16fb jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -206,11 +212,11 @@ templates: {% if shuffled_order == 0 %} - A + 1 {% else %} - B + 2 {% endif %}' metadata: !TemplateMetadata @@ -218,64 +224,58 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: random_choice_1_2 reference: '' - b57e5273-8571-48a3-81a4-18677792f89b: !Template - answer_choices: A ||| B - id: b57e5273-8571-48a3-81a4-18677792f89b - jinja: '{% set shuffled_order = [0, 1] | random %} - - Read the following two sentences. - - {% if shuffled_order == 0 %} - - Sentence A: {{ sentence_good }} - - Sentence B: {{ sentence_bad }} - - {% else %} - - Sentence A: {{ sentence_bad }} - - Sentence B: {{ sentence_good }} - - {% endif %} - - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} - - {{ answer_choices[0] }} - - {% else %} + cefda071-1c35-4fce-9577-c63c7f3f0fde: !Template + answer_choices: Yes ||| No + id: cefda071-1c35-4fce-9577-c63c7f3f0fde + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - {{ answer_choices[1] }} + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' + reference: '' + ddd7ada0-6cb7-448a-85c4-fe3505509bcc: !Template + answer_choices: Yes ||| No + id: ddd7ada0-6cb7-448a-85c4-fe3505509bcc + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - {% endif %}' + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: A/B choice randomized + original_task: false + name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' reference: '' - b772b1a9-d804-4a3d-8767-7d8396fb3e24: !Template + e6ae10f4-f1dd-4733-a806-9e96664ed8ed: !Template answer_choices: null - id: b772b1a9-d804-4a3d-8767-7d8396fb3e24 + id: e6ae10f4-f1dd-4733-a806-9e96664ed8ed jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -283,11 +283,11 @@ templates: {% if shuffled_order == 0 %} - A + 1 {% else %} - B + 2 {% endif %}' metadata: !TemplateMetadata @@ -295,5 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: random_choice_1_2_2 reference: '' diff --git a/promptsource/templates/blimp/irregular_past_participle_adjectives/templates.yaml b/promptsource/templates/blimp/irregular_past_participle_adjectives/templates.yaml index 2a8f0d4e7..2dd39a9bc 100644 --- a/promptsource/templates/blimp/irregular_past_participle_adjectives/templates.yaml +++ b/promptsource/templates/blimp/irregular_past_participle_adjectives/templates.yaml @@ -1,36 +1,35 @@ dataset: blimp subset: irregular_past_participle_adjectives templates: - 02dc7353-0db6-486a-aa42-d00687c9a76b: !Template - answer_choices: null - id: 02dc7353-0db6-486a-aa42-d00687c9a76b + 1a01ffb7-ba38-4dcd-95fe-154fd1f2e1e6: !Template + answer_choices: A ||| B + id: 1a01ffb7-ba38-4dcd-95fe-154fd1f2e1e6 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -38,26 +37,26 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: 'A/B choice randomized (choice order: A-B)' reference: '' - 0806fff0-7705-4703-a2a3-09c1c5eccd88: !Template + 4091948a-5bcd-4dee-96c7-8d01097cc60d: !Template answer_choices: null - id: 0806fff0-7705-4703-a2a3-09c1c5eccd88 + id: 4091948a-5bcd-4dee-96c7-8d01097cc60d jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -65,11 +64,11 @@ templates: {% if shuffled_order == 0 %} - A + 1 {% else %} - B + 2 {% endif %}' metadata: !TemplateMetadata @@ -77,11 +76,27 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: random_choice_1_2 + reference: '' + 47d0cdaf-8702-42f6-b998-919b845a310e: !Template + answer_choices: Yes ||| No + id: 47d0cdaf-8702-42f6-b998-919b845a310e + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' reference: '' - 0850c29e-05ab-4a0e-b05a-46d77a610ebf: !Template + 7375cdca-4772-4e31-9e7e-e1b7bf5c09b2: !Template answer_choices: null - id: 0850c29e-05ab-4a0e-b05a-46d77a610ebf + id: 7375cdca-4772-4e31-9e7e-e1b7bf5c09b2 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -118,9 +133,25 @@ templates: original_task: true name: random_choice_1_2_2 reference: '' - 1b6f1d5e-6285-4e26-9cf3-88c6c4e00874: !Template + 7a89daa4-c45e-4d7b-ad7c-55de85a2ffcd: !Template + answer_choices: Yes ||| No + id: 7a89daa4-c45e-4d7b-ad7c-55de85a2ffcd + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' + reference: '' + a292d01a-f262-449f-b1d4-979fcfcb4ad2: !Template answer_choices: null - id: 1b6f1d5e-6285-4e26-9cf3-88c6c4e00874 + id: a292d01a-f262-449f-b1d4-979fcfcb4ad2 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer A or B. @@ -157,51 +188,29 @@ templates: original_task: true name: random_choice_A_B_2 reference: '' - 23cbd23b-8e98-4c4f-8730-022a6dd1cadb: !Template - answer_choices: A ||| B - id: 23cbd23b-8e98-4c4f-8730-022a6dd1cadb - jinja: '{% set shuffled_order = [0, 1] | random %} - - Read the following two sentences. - - {% if shuffled_order == 0 %} - - Sentence A: {{ sentence_good }} - - Sentence B: {{ sentence_bad }} - - {% else %} - - Sentence A: {{ sentence_bad }} - - Sentence B: {{ sentence_good }} - - {% endif %} - - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} + a732517d-7970-4640-9e2c-f66866adbc02: !Template + answer_choices: Yes ||| No + id: a732517d-7970-4640-9e2c-f66866adbc02 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - {% endif %}' + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: A/B choice randomized (choice order reversed) + original_task: false + name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' reference: '' - 278c50d2-ea29-4ea7-95c8-0ff8a95e943a: !Template + b8a31816-9fa3-483d-ae32-5e694e5c7e1c: !Template answer_choices: Yes ||| No - id: 278c50d2-ea29-4ea7-95c8-0ff8a95e943a + id: b8a31816-9fa3-483d-ae32-5e694e5c7e1c jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata @@ -209,11 +218,11 @@ templates: metrics: - Accuracy original_task: false - name: Yes/No binary judgment for bad sentences + name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' reference: '' - 2e9e9051-6f10-4862-8328-2db12b1d85e8: !Template + bd4ef1c2-c5eb-420b-897a-ee4d2102657d: !Template answer_choices: A ||| B - id: 2e9e9051-6f10-4862-8328-2db12b1d85e8 + id: bd4ef1c2-c5eb-420b-897a-ee4d2102657d jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -232,7 +241,7 @@ templates: {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order + Which one is a better sentence of English, B or A? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -247,53 +256,44 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized + name: 'A/B choice randomized (choice order: B-A)' reference: '' - 3a8e9ab1-0304-4813-82b5-b3635ada8862: !Template - answer_choices: Yes ||| No - id: 3a8e9ab1-0304-4813-82b5-b3635ada8862 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + bedfb44d-0bdd-469e-a57b-ab2c531edd45: !Template + answer_choices: null + id: bedfb44d-0bdd-469e-a57b-ab2c531edd45 + jinja: '{% set shuffled_order = [0, 1] | random %} - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences - reference: '' - 4f93965c-9453-4d4c-8a31-e229a0c49a23: !Template - answer_choices: Yes ||| No - id: 4f93965c-9453-4d4c-8a31-e229a0c49a23 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + Which one of the following sentences is grammatical? Please answer A or B. - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences (choice order reversed) - reference: '' - 9bc7df13-b5a9-453b-a286-c6fb8509b58f: !Template - answer_choices: Yes ||| No - id: 9bc7df13-b5a9-453b-a286-c6fb8509b58f - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + {% if shuffled_order == 0 %} - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences (choice order reversed) + original_task: true + name: random_choice_A_B reference: '' diff --git a/promptsource/templates/blimp/irregular_past_participle_verbs/templates.yaml b/promptsource/templates/blimp/irregular_past_participle_verbs/templates.yaml index 3ea908ea9..35aa8daae 100644 --- a/promptsource/templates/blimp/irregular_past_participle_verbs/templates.yaml +++ b/promptsource/templates/blimp/irregular_past_participle_verbs/templates.yaml @@ -1,28 +1,28 @@ dataset: blimp subset: irregular_past_participle_verbs templates: - 1d8c20e1-b160-4500-b0b2-99d91f09d718: !Template + 05d49540-bde0-46b8-801d-b8cde4c28e7f: !Template answer_choices: Yes ||| No - id: 1d8c20e1-b160-4500-b0b2-99d91f09d718 + id: 05d49540-bde0-46b8-801d-b8cde4c28e7f jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: Yes/No binary judgment for good sentences + name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' reference: '' - 2207891a-b466-449c-8a79-15c5b97ec5ae: !Template + 07f426ef-3848-4175-91b5-070199cff944: !Template answer_choices: null - id: 2207891a-b466-449c-8a79-15c5b97ec5ae + id: 07f426ef-3848-4175-91b5-070199cff944 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -54,14 +54,14 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: random_choice_A_B reference: '' - 375f0be0-e7ac-4c80-a9db-e279c42f4f27: !Template + 39797145-c279-4a20-9422-842a1af6e139: !Template answer_choices: null - id: 375f0be0-e7ac-4c80-a9db-e279c42f4f27 + id: 39797145-c279-4a20-9422-842a1af6e139 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -93,15 +93,15 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: random_choice_1_2 reference: '' - 40b0b799-f6a6-4f67-aaf2-f9ddebe3fe32: !Template + 6bdb527d-4ac2-4aac-bf46-52d823318b10: !Template answer_choices: Yes ||| No - id: 40b0b799-f6a6-4f67-aaf2-f9ddebe3fe32 + id: 6bdb527d-4ac2-4aac-bf46-52d823318b10 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -109,11 +109,11 @@ templates: metrics: - Accuracy original_task: false - name: Yes/No binary judgment for good sentences (choice order reversed) + name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' reference: '' - 4896f5d2-f044-4a4e-a222-329c2740d2af: !Template + 98bc55d7-093f-44ba-bdf4-7481579675db: !Template answer_choices: A ||| B - id: 4896f5d2-f044-4a4e-a222-329c2740d2af + id: 98bc55d7-093f-44ba-bdf4-7481579675db jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -132,7 +132,7 @@ templates: {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order + Which one is a better sentence of English, A or B? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -147,14 +147,14 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized (choice order reversed) + name: 'A/B choice randomized (choice order: A-B)' reference: '' - 6351922c-0b3f-4e0c-bee9-84052a48861e: !Template + a50db2db-154e-4e0b-87f3-5c1a453e39ff: !Template answer_choices: null - id: 6351922c-0b3f-4e0c-bee9-84052a48861e + id: a50db2db-154e-4e0b-87f3-5c1a453e39ff jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -186,38 +186,53 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: random_choice_A_B_2 reference: '' - 74a843fc-2f3a-408d-8717-fba1fd4a9a81: !Template - answer_choices: null - id: 74a843fc-2f3a-408d-8717-fba1fd4a9a81 + b1bb44fc-fb4d-49a4-b149-21bd6b2e0432: !Template + answer_choices: Yes ||| No + id: b1bb44fc-fb4d-49a4-b149-21bd6b2e0432 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' + reference: '' + bfb444a9-cc45-47dc-b041-c75713b996bc: !Template + answer_choices: A ||| B + id: bfb444a9-cc45-47dc-b041-c75713b996bc jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -225,69 +240,38 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 - reference: '' - c2b54abf-0de1-4be0-ade2-998e3b7d3d2e: !Template - answer_choices: Yes ||| No - id: c2b54abf-0de1-4be0-ade2-998e3b7d3d2e - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences + name: 'A/B choice randomized (choice order: B-A)' reference: '' - c909bbc2-fc9c-4d05-9367-cce7ad261b4f: !Template - answer_choices: Yes ||| No - id: c909bbc2-fc9c-4d05-9367-cce7ad261b4f - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences (choice order reversed) - reference: '' - f5f2ac7f-4daa-48f2-b376-2092bd7044f5: !Template - answer_choices: A ||| B - id: f5f2ac7f-4daa-48f2-b376-2092bd7044f5 + d54c908d-ad8b-4c97-8f12-d903a2be93e8: !Template + answer_choices: null + id: d54c908d-ad8b-4c97-8f12-d903a2be93e8 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| - {{ answer_choices[0] }} + {% if shuffled_order == 0 %} + + 1 {% else %} - {{ answer_choices[1] }} + 2 {% endif %}' metadata: !TemplateMetadata @@ -295,5 +279,21 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized + name: random_choice_1_2_2 + reference: '' + db009dc0-b4c3-4067-a421-9a68cdf96bad: !Template + answer_choices: Yes ||| No + id: db009dc0-b4c3-4067-a421-9a68cdf96bad + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' reference: '' diff --git a/promptsource/templates/blimp/irregular_plural_subject_verb_agreement_1/templates.yaml b/promptsource/templates/blimp/irregular_plural_subject_verb_agreement_1/templates.yaml index 4908ee1f7..db5c18280 100644 --- a/promptsource/templates/blimp/irregular_plural_subject_verb_agreement_1/templates.yaml +++ b/promptsource/templates/blimp/irregular_plural_subject_verb_agreement_1/templates.yaml @@ -1,25 +1,9 @@ dataset: blimp subset: irregular_plural_subject_verb_agreement_1 templates: - 07969670-5712-41fc-ae94-23a48da04c23: !Template + 0f0fd016-1f19-4715-8e05-9ff549088767: !Template answer_choices: Yes ||| No - id: 07969670-5712-41fc-ae94-23a48da04c23 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences (choice order reversed) - reference: '' - 138928eb-3b09-42f8-9e6b-f45b6533919d: !Template - answer_choices: Yes ||| No - id: 138928eb-3b09-42f8-9e6b-f45b6533919d + id: 0f0fd016-1f19-4715-8e05-9ff549088767 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -31,26 +15,26 @@ templates: metrics: - Accuracy original_task: false - name: Yes/No binary judgment for bad sentences + name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' reference: '' - 15d22e3c-1265-4ce4-98df-9236e34bca27: !Template + 222939ba-e155-4ff3-9ae7-ff1cd5c2713e: !Template answer_choices: null - id: 15d22e3c-1265-4ce4-98df-9236e34bca27 + id: 222939ba-e155-4ff3-9ae7-ff1cd5c2713e jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -58,11 +42,11 @@ templates: {% if shuffled_order == 0 %} - 1 + A {% else %} - 2 + B {% endif %}' metadata: !TemplateMetadata @@ -70,38 +54,37 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: random_choice_A_B_2 reference: '' - 32abd943-3331-45dc-ba1f-00b8603cac53: !Template - answer_choices: null - id: 32abd943-3331-45dc-ba1f-00b8603cac53 + 3506c95f-a239-4eaf-ad92-62e9ff23d3db: !Template + answer_choices: A ||| B + id: 3506c95f-a239-4eaf-ad92-62e9ff23d3db jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -109,11 +92,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: 'A/B choice randomized (choice order: A-B)' reference: '' - 5e1bb1ec-ee7b-4535-83e6-e5bb23f4ca44: !Template + 355c9af4-4aff-4039-b35d-b9fc04893375: !Template answer_choices: A ||| B - id: 5e1bb1ec-ee7b-4535-83e6-e5bb23f4ca44 + id: 355c9af4-4aff-4039-b35d-b9fc04893375 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -147,26 +130,42 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized (choice order reversed) + name: 'A/B choice randomized (choice order: B-A)' reference: '' - 732e9a51-add7-4726-aea6-a133527cec8b: !Template + 523dcb7c-8a66-46de-8d80-0d18f53583b5: !Template + answer_choices: Yes ||| No + id: 523dcb7c-8a66-46de-8d80-0d18f53583b5 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' + reference: '' + 56d641b8-b550-454a-8b50-2b8637ed78b0: !Template answer_choices: null - id: 732e9a51-add7-4726-aea6-a133527cec8b + id: 56d641b8-b550-454a-8b50-2b8637ed78b0 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -174,11 +173,11 @@ templates: {% if shuffled_order == 0 %} - A + 1 {% else %} - B + 2 {% endif %}' metadata: !TemplateMetadata @@ -186,37 +185,54 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: random_choice_1_2_2 reference: '' - bc17aab3-1102-4e38-899f-b8f8a10c2699: !Template - answer_choices: A ||| B - id: bc17aab3-1102-4e38-899f-b8f8a10c2699 + 5fcd802b-29d4-417c-9871-6e7337f56f2c: !Template + answer_choices: Yes ||| No + id: 5fcd802b-29d4-417c-9871-6e7337f56f2c + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' + reference: '' + 613c8f2e-8c3d-41f2-b1bc-d2fd705ef80d: !Template + answer_choices: null + id: 613c8f2e-8c3d-41f2-b1bc-d2fd705ef80d jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| - {{ answer_choices[0] }} + {% if shuffled_order == 0 %} + + A {% else %} - {{ answer_choices[1] }} + B {% endif %}' metadata: !TemplateMetadata @@ -224,42 +240,26 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized - reference: '' - cb17bb57-f035-4a9b-afc0-e010634d4c51: !Template - answer_choices: Yes ||| No - id: cb17bb57-f035-4a9b-afc0-e010634d4c51 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences + name: random_choice_A_B reference: '' - cc7203ea-8c85-40a8-a778-baa94467b25c: !Template + 619c0b3d-d914-42f3-af5a-3da0ed47fd61: !Template answer_choices: null - id: cc7203ea-8c85-40a8-a778-baa94467b25c + id: 619c0b3d-d914-42f3-af5a-3da0ed47fd61 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -267,11 +267,11 @@ templates: {% if shuffled_order == 0 %} - A + 1 {% else %} - B + 2 {% endif %}' metadata: !TemplateMetadata @@ -279,21 +279,21 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: random_choice_1_2 reference: '' - d0884cf3-1420-4064-981d-7daebe5b933a: !Template + 6cc4b448-6fb7-4385-baba-6ff904dfcc9e: !Template answer_choices: Yes ||| No - id: d0884cf3-1420-4064-981d-7daebe5b933a + id: 6cc4b448-6fb7-4385-baba-6ff904dfcc9e jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: Yes/No binary judgment for good sentences (choice order reversed) + name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' reference: '' diff --git a/promptsource/templates/blimp/irregular_plural_subject_verb_agreement_2/templates.yaml b/promptsource/templates/blimp/irregular_plural_subject_verb_agreement_2/templates.yaml index 09acd2d89..1f09dec8d 100644 --- a/promptsource/templates/blimp/irregular_plural_subject_verb_agreement_2/templates.yaml +++ b/promptsource/templates/blimp/irregular_plural_subject_verb_agreement_2/templates.yaml @@ -1,9 +1,9 @@ dataset: blimp subset: irregular_plural_subject_verb_agreement_2 templates: - 29407110-06ab-4aab-8c8b-2d025da8bcad: !Template + 14f0a497-ffae-486e-87b9-e114a4395ada: !Template answer_choices: A ||| B - id: 29407110-06ab-4aab-8c8b-2d025da8bcad + id: 14f0a497-ffae-486e-87b9-e114a4395ada jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -37,30 +37,14 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized (choice order reversed) + name: 'A/B choice randomized (choice order: B-A)' reference: '' - 2ef81a47-e0e5-43ac-84a6-4587760a3da9: !Template - answer_choices: Yes ||| No - id: 2ef81a47-e0e5-43ac-84a6-4587760a3da9 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences (choice order reversed) - reference: '' - 46512145-cd20-4d93-bba7-d3e403683925: !Template + 20f3b2ee-426e-4999-aa43-ae903a028348: !Template answer_choices: null - id: 46512145-cd20-4d93-bba7-d3e403683925 + id: 20f3b2ee-426e-4999-aa43-ae903a028348 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -92,50 +76,27 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: random_choice_A_B reference: '' - 5e1a0869-264a-4870-804b-542499923f9f: !Template - answer_choices: null - id: 5e1a0869-264a-4870-804b-542499923f9f - jinja: '{% set shuffled_order = [0, 1] | random %} - - Which one of the following sentences is grammatical? Please answer 1 or 2. - - {% if shuffled_order == 0 %} - - 1: {{ sentence_good }} - - 2: {{ sentence_bad }} - - {% else %} - - 1: {{ sentence_bad }} - - 2: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - 1 - - {% else %} - - 2 + 3e5bec52-b152-495e-8bf0-f6462997a455: !Template + answer_choices: Yes ||| No + id: 3e5bec52-b152-495e-8bf0-f6462997a455 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - {% endif %}' + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: random_choice_1_2 + original_task: false + name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' reference: '' - 65b77969-a0cf-472c-9c0b-437acae763a7: !Template + 4d03ecc5-9fee-487e-bd8d-b079731cc239: !Template answer_choices: null - id: 65b77969-a0cf-472c-9c0b-437acae763a7 + id: 4d03ecc5-9fee-487e-bd8d-b079731cc239 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -172,12 +133,12 @@ templates: original_task: true name: random_choice_1_2_2 reference: '' - 7ed1f1fe-25e7-43eb-abd8-9dcc1146d0f2: !Template + 59cc4a6a-178d-4cce-93dd-d586a810056e: !Template answer_choices: null - id: 7ed1f1fe-25e7-43eb-abd8-9dcc1146d0f2 + id: 59cc4a6a-178d-4cce-93dd-d586a810056e jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -209,11 +170,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: random_choice_A_B_2 reference: '' - 88d5eb31-2be6-44ff-82be-5263900c5159: !Template + 7e00c53f-3cc4-4747-b501-795a3f016966: !Template answer_choices: A ||| B - id: 88d5eb31-2be6-44ff-82be-5263900c5159 + id: 7e00c53f-3cc4-4747-b501-795a3f016966 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -247,15 +208,15 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized + name: 'A/B choice randomized (choice order: A-B)' reference: '' - 94ca08bd-aa2d-4317-8366-dd85932d455a: !Template + 7eded14d-5d1b-40a4-ba11-a9d45feaaa79: !Template answer_choices: Yes ||| No - id: 94ca08bd-aa2d-4317-8366-dd85932d455a + id: 7eded14d-5d1b-40a4-ba11-a9d45feaaa79 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -263,27 +224,27 @@ templates: metrics: - Accuracy original_task: false - name: Yes/No binary judgment for good sentences + name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' reference: '' - a6701d62-53f9-4fad-ac3c-745a8842fa59: !Template + 7f7484bc-3faf-4462-b453-321c0943b241: !Template answer_choices: Yes ||| No - id: a6701d62-53f9-4fad-ac3c-745a8842fa59 + id: 7f7484bc-3faf-4462-b453-321c0943b241 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: Yes/No binary judgment for good sentences (choice order reversed) + name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' reference: '' - fa23d667-6719-4b80-94c1-0cd563c8fc2d: !Template + aee62b04-04c9-4267-ab36-03e1b79415b2: !Template answer_choices: Yes ||| No - id: fa23d667-6719-4b80-94c1-0cd563c8fc2d + id: aee62b04-04c9-4267-ab36-03e1b79415b2 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -295,5 +256,44 @@ templates: metrics: - Accuracy original_task: false - name: Yes/No binary judgment for bad sentences + name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' + reference: '' + e7ea957b-df45-4e9b-804e-975e9ea50c8d: !Template + answer_choices: null + id: e7ea957b-df45-4e9b-804e-975e9ea50c8d + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2 reference: '' diff --git a/promptsource/templates/blimp/left_branch_island_echo_question/templates.yaml b/promptsource/templates/blimp/left_branch_island_echo_question/templates.yaml index 723796ce3..7d55c8625 100644 --- a/promptsource/templates/blimp/left_branch_island_echo_question/templates.yaml +++ b/promptsource/templates/blimp/left_branch_island_echo_question/templates.yaml @@ -1,36 +1,51 @@ dataset: blimp subset: left_branch_island_echo_question templates: - 0eceb998-0f5d-4bb5-b540-3233d7f698cb: !Template - answer_choices: null - id: 0eceb998-0f5d-4bb5-b540-3233d7f698cb + 0cf0efad-670f-4dc2-95a5-6a151ce47bec: !Template + answer_choices: Yes ||| No + id: 0cf0efad-670f-4dc2-95a5-6a151ce47bec + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' + reference: '' + 3be69c93-e8a3-4204-9d62-325c6e50398a: !Template + answer_choices: A ||| B + id: 3be69c93-e8a3-4204-9d62-325c6e50398a jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -38,86 +53,37 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 - reference: '' - 1c8d3485-79db-4d64-986b-87b885b3daa2: !Template - answer_choices: Yes ||| No - id: 1c8d3485-79db-4d64-986b-87b885b3daa2 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences (choice order reversed) + name: 'A/B choice randomized (choice order: A-B)' reference: '' - 1d543084-035c-45a5-a09c-21dd4678059c: !Template - answer_choices: Yes ||| No - id: 1d543084-035c-45a5-a09c-21dd4678059c - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences - reference: '' - 4383c751-d28e-4bb3-a7cc-eed29a3a984d: !Template - answer_choices: Yes ||| No - id: 4383c751-d28e-4bb3-a7cc-eed29a3a984d - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences - reference: '' - 778bad58-dcc3-4dd6-b384-465094df51bd: !Template - answer_choices: null - id: 778bad58-dcc3-4dd6-b384-465094df51bd + 451b85b0-b3ad-4279-b4e7-62f7a0ee4d5a: !Template + answer_choices: A ||| B + id: 451b85b0-b3ad-4279-b4e7-62f7a0ee4d5a jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -125,11 +91,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: 'A/B choice randomized (choice order: B-A)' reference: '' - 921d7113-a724-4849-8f92-af8cf83e062c: !Template + 5f40393a-be79-4dad-98bb-30f0144c5b2e: !Template answer_choices: null - id: 921d7113-a724-4849-8f92-af8cf83e062c + id: 5f40393a-be79-4dad-98bb-30f0144c5b2e jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer A or B. @@ -166,9 +132,25 @@ templates: original_task: true name: random_choice_A_B reference: '' - a8e82ebd-d361-4129-9bba-04ab08545c9b: !Template + 70bcf6ab-3460-44d5-acba-0f5648868f9e: !Template + answer_choices: Yes ||| No + id: 70bcf6ab-3460-44d5-acba-0f5648868f9e + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' + reference: '' + 80f41567-8aef-4c05-867b-5555912d6c7d: !Template answer_choices: null - id: a8e82ebd-d361-4129-9bba-04ab08545c9b + id: 80f41567-8aef-4c05-867b-5555912d6c7d jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer 1 or 2. @@ -205,51 +187,36 @@ templates: original_task: true name: random_choice_1_2 reference: '' - b58d6246-9221-43fe-bea5-4ff966cb8563: !Template - answer_choices: Yes ||| No - id: b58d6246-9221-43fe-bea5-4ff966cb8563 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences (choice order reversed) - reference: '' - ba91c932-61de-4b29-b2d7-551f3f0dfa8f: !Template - answer_choices: A ||| B - id: ba91c932-61de-4b29-b2d7-551f3f0dfa8f + c15a84e8-96d6-48c7-aa02-83f5e935c9fc: !Template + answer_choices: null + id: c15a84e8-96d6-48c7-aa02-83f5e935c9fc jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| - {{ answer_choices[0] }} + {% if shuffled_order == 0 %} + + A {% else %} - {{ answer_choices[1] }} + B {% endif %}' metadata: !TemplateMetadata @@ -257,37 +224,54 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized + name: random_choice_A_B_2 reference: '' - d73bced1-ba60-48ae-b958-a8fb818bfc68: !Template - answer_choices: A ||| B - id: d73bced1-ba60-48ae-b958-a8fb818bfc68 + c9ae4c70-f150-401a-bfa7-67794a694d0b: !Template + answer_choices: Yes ||| No + id: c9ae4c70-f150-401a-bfa7-67794a694d0b + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' + reference: '' + dafde0f0-b07a-435c-be05-1d28a3a26edc: !Template + answer_choices: null + id: dafde0f0-b07a-435c-be05-1d28a3a26edc jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| - {{ answer_choices[0] }} + {% if shuffled_order == 0 %} + + 1 {% else %} - {{ answer_choices[1] }} + 2 {% endif %}' metadata: !TemplateMetadata @@ -295,5 +279,21 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized (choice order reversed) + name: random_choice_1_2_2 + reference: '' + e039d66f-3dfa-4451-9102-055ad545f55b: !Template + answer_choices: Yes ||| No + id: e039d66f-3dfa-4451-9102-055ad545f55b + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' reference: '' diff --git a/promptsource/templates/blimp/left_branch_island_simple_question/templates.yaml b/promptsource/templates/blimp/left_branch_island_simple_question/templates.yaml index fa8c761f4..81f03a0bf 100644 --- a/promptsource/templates/blimp/left_branch_island_simple_question/templates.yaml +++ b/promptsource/templates/blimp/left_branch_island_simple_question/templates.yaml @@ -1,12 +1,60 @@ dataset: blimp subset: left_branch_island_simple_question templates: - 3993c5b2-dd53-4412-892b-78c84241d2dc: !Template + 17335753-95c8-4d9f-ad1a-a13234d930aa: !Template + answer_choices: Yes ||| No + id: 17335753-95c8-4d9f-ad1a-a13234d930aa + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' + reference: '' + 1b58fda4-95bc-44f6-a4cb-92c2f91d313b: !Template + answer_choices: Yes ||| No + id: 1b58fda4-95bc-44f6-a4cb-92c2f91d313b + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' + reference: '' + 5967ff7d-1b12-463c-b8e5-a8e20a4bfeba: !Template + answer_choices: Yes ||| No + id: 5967ff7d-1b12-463c-b8e5-a8e20a4bfeba + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' + reference: '' + 614f74b1-9130-4c89-b65b-8e8efa3069bd: !Template answer_choices: null - id: 3993c5b2-dd53-4412-892b-78c84241d2dc + id: 614f74b1-9130-4c89-b65b-8e8efa3069bd jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -38,37 +86,38 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: random_choice_A_B_2 reference: '' - 41b1a01f-2115-4303-bb4d-95431bafe7f1: !Template - answer_choices: A ||| B - id: 41b1a01f-2115-4303-bb4d-95431bafe7f1 + 7fec3889-4649-4c78-8633-430c3683c847: !Template + answer_choices: null + id: 7fec3889-4649-4c78-8633-430c3683c847 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| - {{ answer_choices[0] }} + {% if shuffled_order == 0 %} + + A {% else %} - {{ answer_choices[1] }} + B {% endif %}' metadata: !TemplateMetadata @@ -76,38 +125,37 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized + name: random_choice_A_B reference: '' - 66eb418d-7e90-4d03-bff7-58df497417ee: !Template - answer_choices: null - id: 66eb418d-7e90-4d03-bff7-58df497417ee + 830edf75-0c8d-4ed3-ada7-af7b6715fb24: !Template + answer_choices: A ||| B + id: 830edf75-0c8d-4ed3-ada7-af7b6715fb24 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -115,27 +163,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 - reference: '' - 6e3c956d-f4aa-45aa-9f37-c0d61dac5976: !Template - answer_choices: Yes ||| No - id: 6e3c956d-f4aa-45aa-9f37-c0d61dac5976 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences (choice order reversed) + name: 'A/B choice randomized (choice order: B-A)' reference: '' - 6fa62282-0461-458a-9ade-1b146097f5ee: !Template + 99875376-fa46-454a-a2bb-0ea56e0beb21: !Template answer_choices: Yes ||| No - id: 6fa62282-0461-458a-9ade-1b146097f5ee + id: 99875376-fa46-454a-a2bb-0ea56e0beb21 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -147,11 +179,11 @@ templates: metrics: - Accuracy original_task: false - name: Yes/No binary judgment for bad sentences (choice order reversed) + name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' reference: '' - 8eb6c647-eb5e-46f5-9bd8-f17d792be791: !Template + a91945c6-5318-4154-866e-85904700b684: !Template answer_choices: A ||| B - id: 8eb6c647-eb5e-46f5-9bd8-f17d792be791 + id: a91945c6-5318-4154-866e-85904700b684 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -170,7 +202,7 @@ templates: {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order + Which one is a better sentence of English, A or B? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -185,11 +217,11 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized (choice order reversed) + name: 'A/B choice randomized (choice order: A-B)' reference: '' - 935b11ae-afd6-44bf-8fa8-42f0650795d4: !Template + e336851f-5d24-496c-90e1-441f602384eb: !Template answer_choices: null - id: 935b11ae-afd6-44bf-8fa8-42f0650795d4 + id: e336851f-5d24-496c-90e1-441f602384eb jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -226,25 +258,9 @@ templates: original_task: true name: random_choice_1_2_2 reference: '' - cb3a865e-4f7c-4595-be43-c6586608bdb8: !Template - answer_choices: Yes ||| No - id: cb3a865e-4f7c-4595-be43-c6586608bdb8 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences - reference: '' - e9b6032e-6723-454c-be08-15f45d3471dd: !Template + fdaa2577-c1ea-4b60-bfa2-3c58cca94bfc: !Template answer_choices: null - id: e9b6032e-6723-454c-be08-15f45d3471dd + id: fdaa2577-c1ea-4b60-bfa2-3c58cca94bfc jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer 1 or 2. @@ -281,19 +297,3 @@ templates: original_task: true name: random_choice_1_2 reference: '' - fa93c786-5e75-4039-86f6-d7a67423a2bd: !Template - answer_choices: Yes ||| No - id: fa93c786-5e75-4039-86f6-d7a67423a2bd - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences - reference: '' diff --git a/promptsource/templates/blimp/matrix_question_npi_licensor_present/templates.yaml b/promptsource/templates/blimp/matrix_question_npi_licensor_present/templates.yaml index cf91bf26c..e8c7851df 100644 --- a/promptsource/templates/blimp/matrix_question_npi_licensor_present/templates.yaml +++ b/promptsource/templates/blimp/matrix_question_npi_licensor_present/templates.yaml @@ -1,52 +1,51 @@ dataset: blimp subset: matrix_question_npi_licensor_present templates: - 0b1142ff-05ec-4e72-98b5-690a28ef370d: !Template + 1c40ac09-5369-4a9e-a44c-14319215613b: !Template answer_choices: Yes ||| No - id: 0b1142ff-05ec-4e72-98b5-690a28ef370d + id: 1c40ac09-5369-4a9e-a44c-14319215613b jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: Yes/No binary judgment for bad sentences (choice order reversed) + name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' reference: '' - 0f3a0902-5ddd-4097-bf93-db6c8e1d50ec: !Template - answer_choices: null - id: 0f3a0902-5ddd-4097-bf93-db6c8e1d50ec + 28e46e34-0ee1-481b-93f6-010fd2a1c144: !Template + answer_choices: A ||| B + id: 28e46e34-0ee1-481b-93f6-010fd2a1c144 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -54,54 +53,37 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 - reference: '' - 3fb53f8e-11cb-42ca-8c78-8587dc312f0a: !Template - answer_choices: Yes ||| No - id: 3fb53f8e-11cb-42ca-8c78-8587dc312f0a - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences + name: 'A/B choice randomized (choice order: B-A)' reference: '' - 5dfb1a1d-b765-4b98-b86f-738cd92d9d25: !Template - answer_choices: null - id: 5dfb1a1d-b765-4b98-b86f-738cd92d9d25 + 387892c8-c6fc-4110-a460-096ced7092a5: !Template + answer_choices: A ||| B + id: 387892c8-c6fc-4110-a460-096ced7092a5 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -109,11 +91,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: 'A/B choice randomized (choice order: A-B)' reference: '' - 62edcb10-abb0-47df-8102-10cc3be331f7: !Template + 62df645b-2a76-431b-92f3-8c2793701300: !Template answer_choices: Yes ||| No - id: 62edcb10-abb0-47df-8102-10cc3be331f7 + id: 62df645b-2a76-431b-92f3-8c2793701300 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -125,11 +107,43 @@ templates: metrics: - Accuracy original_task: false - name: Yes/No binary judgment for good sentences (choice order reversed) + name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' + reference: '' + 63ad3ddd-d5eb-4e55-9830-c0dedae59f86: !Template + answer_choices: Yes ||| No + id: 63ad3ddd-d5eb-4e55-9830-c0dedae59f86 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' reference: '' - 65f75f9e-3636-4fd6-a406-b9d0d44897c2: !Template + beea3fb1-eb4e-48d0-aab6-5b26f8e606e2: !Template + answer_choices: Yes ||| No + id: beea3fb1-eb4e-48d0-aab6-5b26f8e606e2 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' + reference: '' + c05dd57c-7874-4f05-a5f2-0794ed167cdf: !Template answer_choices: null - id: 65f75f9e-3636-4fd6-a406-b9d0d44897c2 + id: c05dd57c-7874-4f05-a5f2-0794ed167cdf jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -166,35 +180,36 @@ templates: original_task: true name: random_choice_1_2_2 reference: '' - 91f671f7-52e2-432d-9bcc-18db6a5ed691: !Template - answer_choices: A ||| B - id: 91f671f7-52e2-432d-9bcc-18db6a5ed691 + ccde093e-c789-459d-bed9-4967b9b359a4: !Template + answer_choices: null + id: ccde093e-c789-459d-bed9-4967b9b359a4 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| - {{ answer_choices[0] }} + {% if shuffled_order == 0 %} + + A {% else %} - {{ answer_choices[1] }} + B {% endif %}' metadata: !TemplateMetadata @@ -202,30 +217,14 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized (choice order reversed) - reference: '' - a401118b-af0e-4f52-ac3a-b057acd65d1e: !Template - answer_choices: Yes ||| No - id: a401118b-af0e-4f52-ac3a-b057acd65d1e - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences + name: random_choice_A_B_2 reference: '' - a91a1346-df74-4f7d-b188-6fbcd472a9b0: !Template + d60097f5-215c-43cc-a78f-9ccca5db2adc: !Template answer_choices: null - id: a91a1346-df74-4f7d-b188-6fbcd472a9b0 + id: d60097f5-215c-43cc-a78f-9ccca5db2adc jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -257,37 +256,38 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: random_choice_A_B reference: '' - cf4b9b83-cc0f-4c2e-b923-06f099b56c37: !Template - answer_choices: A ||| B - id: cf4b9b83-cc0f-4c2e-b923-06f099b56c37 + da633e5d-adb4-4027-9204-6bf529dbd199: !Template + answer_choices: null + id: da633e5d-adb4-4027-9204-6bf529dbd199 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| - {{ answer_choices[0] }} + {% if shuffled_order == 0 %} + + 1 {% else %} - {{ answer_choices[1] }} + 2 {% endif %}' metadata: !TemplateMetadata @@ -295,5 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized + name: random_choice_1_2 reference: '' diff --git a/promptsource/templates/blimp/npi_present_1/templates.yaml b/promptsource/templates/blimp/npi_present_1/templates.yaml index e0053b4ca..5dd7e9e0d 100644 --- a/promptsource/templates/blimp/npi_present_1/templates.yaml +++ b/promptsource/templates/blimp/npi_present_1/templates.yaml @@ -1,52 +1,13 @@ dataset: blimp subset: npi_present_1 templates: - 24f2d528-aa35-4f44-bb71-7f8f30716ff5: !Template - answer_choices: null - id: 24f2d528-aa35-4f44-bb71-7f8f30716ff5 - jinja: '{% set shuffled_order = [0, 1] | random %} - - Which one of the following sentences is grammatical? Please answer A or B. - - {% if shuffled_order == 0 %} - - A: {{ sentence_good }} - - B: {{ sentence_bad }} - - {% else %} - - A: {{ sentence_bad }} - - B: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - A - - {% else %} - - B - - {% endif %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: random_choice_A_B - reference: '' - 31bc6a1e-cbed-4b52-9f81-774b035251d4: !Template + 14429731-f9a5-4a5e-8a0e-eddde4cf8e24: !Template answer_choices: Yes ||| No - id: 31bc6a1e-cbed-4b52-9f81-774b035251d4 + id: 14429731-f9a5-4a5e-8a0e-eddde4cf8e24 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata @@ -54,15 +15,15 @@ templates: metrics: - Accuracy original_task: false - name: Yes/No binary judgment for bad sentences + name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' reference: '' - 3c54a3db-f8ea-496c-90d9-80493742adcb: !Template + 1a7d2011-76d3-486d-8ab7-021721cc72a1: !Template answer_choices: Yes ||| No - id: 3c54a3db-f8ea-496c-90d9-80493742adcb + id: 1a7d2011-76d3-486d-8ab7-021721cc72a1 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -70,11 +31,11 @@ templates: metrics: - Accuracy original_task: false - name: Yes/No binary judgment for good sentences (choice order reversed) + name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' reference: '' - 70b6b55d-e052-4664-a60e-8da216aa9b10: !Template + 92d739a8-fda7-4fc7-83be-560c2b913702: !Template answer_choices: A ||| B - id: 70b6b55d-e052-4664-a60e-8da216aa9b10 + id: 92d739a8-fda7-4fc7-83be-560c2b913702 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -108,27 +69,11 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized (choice order reversed) - reference: '' - 76542a82-0e43-47f0-bf91-5908e3baab16: !Template - answer_choices: Yes ||| No - id: 76542a82-0e43-47f0-bf91-5908e3baab16 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences (choice order reversed) + name: 'A/B choice randomized (choice order: B-A)' reference: '' - 7dc5783a-0d77-4c1b-bb3b-9737c45f1860: !Template + 95500764-37bd-466e-9bd3-c4bb179f79c8: !Template answer_choices: null - id: 7dc5783a-0d77-4c1b-bb3b-9737c45f1860 + id: 95500764-37bd-466e-9bd3-c4bb179f79c8 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer 1 or 2. @@ -165,9 +110,9 @@ templates: original_task: true name: random_choice_1_2 reference: '' - bd9bd837-7468-4e93-a022-f2baca56cd62: !Template + bb05b582-e555-4a77-9a1a-c98e58fd5784: !Template answer_choices: A ||| B - id: bd9bd837-7468-4e93-a022-f2baca56cd62 + id: bb05b582-e555-4a77-9a1a-c98e58fd5784 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -201,27 +146,11 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized - reference: '' - ce9ba367-5943-4be0-8768-7a2f739d3715: !Template - answer_choices: Yes ||| No - id: ce9ba367-5943-4be0-8768-7a2f739d3715 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences + name: 'A/B choice randomized (choice order: A-B)' reference: '' - d31d78e4-2de8-4ca1-aecf-6f22a2507ed9: !Template + c1c38791-307e-4aa7-a317-6152d3eafe53: !Template answer_choices: null - id: d31d78e4-2de8-4ca1-aecf-6f22a2507ed9 + id: c1c38791-307e-4aa7-a317-6152d3eafe53 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer A or B. @@ -258,9 +187,9 @@ templates: original_task: true name: random_choice_A_B_2 reference: '' - d838ecfc-afac-4e9a-b28c-10661d610cec: !Template + ce21e2be-b2e3-4920-941e-5e8e5c3eff2d: !Template answer_choices: null - id: d838ecfc-afac-4e9a-b28c-10661d610cec + id: ce21e2be-b2e3-4920-941e-5e8e5c3eff2d jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -297,3 +226,74 @@ templates: original_task: true name: random_choice_1_2_2 reference: '' + dcb03908-30ba-4089-a3ed-9e0754bfee35: !Template + answer_choices: Yes ||| No + id: dcb03908-30ba-4089-a3ed-9e0754bfee35 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' + reference: '' + e062b5fb-0b2d-4de3-8ebf-95a93d2eead5: !Template + answer_choices: null + id: e062b5fb-0b2d-4de3-8ebf-95a93d2eead5 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B + reference: '' + eedf81c3-95e2-4e7e-9dd1-af108079dfa9: !Template + answer_choices: Yes ||| No + id: eedf81c3-95e2-4e7e-9dd1-af108079dfa9 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' + reference: '' diff --git a/promptsource/templates/blimp/npi_present_2/templates.yaml b/promptsource/templates/blimp/npi_present_2/templates.yaml index fc144f35f..06ad994bc 100644 --- a/promptsource/templates/blimp/npi_present_2/templates.yaml +++ b/promptsource/templates/blimp/npi_present_2/templates.yaml @@ -1,24 +1,24 @@ dataset: blimp subset: npi_present_2 templates: - 09929273-9ae4-4470-a164-d4c3cec38d97: !Template + 014167ed-81cc-49dd-889c-b67b902485c0: !Template answer_choices: null - id: 09929273-9ae4-4470-a164-d4c3cec38d97 + id: 014167ed-81cc-49dd-889c-b67b902485c0 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -26,11 +26,11 @@ templates: {% if shuffled_order == 0 %} - A + 1 {% else %} - B + 2 {% endif %}' metadata: !TemplateMetadata @@ -38,26 +38,26 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: random_choice_1_2_2 reference: '' - 32e09079-c0f9-4e4a-8ad6-fb38bc7960e8: !Template + 158c8aab-7b7f-4869-8f01-32c11d450cd3: !Template answer_choices: null - id: 32e09079-c0f9-4e4a-8ad6-fb38bc7960e8 + id: 158c8aab-7b7f-4869-8f01-32c11d450cd3 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -65,11 +65,11 @@ templates: {% if shuffled_order == 0 %} - A + 1 {% else %} - B + 2 {% endif %}' metadata: !TemplateMetadata @@ -77,80 +77,58 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: random_choice_1_2 reference: '' - 4367332e-8220-4ca8-8300-da4ee43ab392: !Template - answer_choices: A ||| B - id: 4367332e-8220-4ca8-8300-da4ee43ab392 - jinja: '{% set shuffled_order = [0, 1] | random %} - - Read the following two sentences. - - {% if shuffled_order == 0 %} - - Sentence A: {{ sentence_good }} - - Sentence B: {{ sentence_bad }} - - {% else %} - - Sentence A: {{ sentence_bad }} - - Sentence B: {{ sentence_good }} - - {% endif %} - - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} + 24b330d2-525f-4f85-bf9b-3910ebe846d4: !Template + answer_choices: Yes ||| No + id: 24b330d2-525f-4f85-bf9b-3910ebe846d4 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - {% endif %}' + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: A/B choice randomized (choice order reversed) + original_task: false + name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' reference: '' - 572e0b15-8fbe-41af-8630-1f94d5a40449: !Template + 4600694f-1d45-474f-a295-557c1d7189b1: !Template answer_choices: Yes ||| No - id: 572e0b15-8fbe-41af-8630-1f94d5a40449 + id: 4600694f-1d45-474f-a295-557c1d7189b1 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: Yes/No binary judgment for bad sentences (choice order reversed) + name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' reference: '' - 77f29518-5934-4aa6-a85c-5f99f99ee644: !Template + 812d1a83-7e31-47fb-97a4-7ac8033404ff: !Template answer_choices: null - id: 77f29518-5934-4aa6-a85c-5f99f99ee644 + id: 812d1a83-7e31-47fb-97a4-7ac8033404ff jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -158,11 +136,11 @@ templates: {% if shuffled_order == 0 %} - 1 + A {% else %} - 2 + B {% endif %}' metadata: !TemplateMetadata @@ -170,11 +148,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: random_choice_A_B_2 reference: '' - 78dd7c42-3402-4ba9-ab13-91ff6c5091fe: !Template + a008869a-8ace-4ba0-a44b-c2a511030c4d: !Template answer_choices: A ||| B - id: 78dd7c42-3402-4ba9-ab13-91ff6c5091fe + id: a008869a-8ace-4ba0-a44b-c2a511030c4d jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -208,47 +186,53 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized + name: 'A/B choice randomized (choice order: A-B)' reference: '' - 9176e7fb-1ded-424a-8263-0f5d67a6e6f3: !Template - answer_choices: Yes ||| No - id: 9176e7fb-1ded-424a-8263-0f5d67a6e6f3 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + dcd5effe-d29d-4f21-b2eb-d2ae5148bd00: !Template + answer_choices: A ||| B + id: dcd5effe-d29d-4f21-b2eb-d2ae5148bd00 + jinja: '{% set shuffled_order = [0, 1] | random %} - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences - reference: '' - b6abf882-a346-4a35-bf34-2067a3a6e9f3: !Template - answer_choices: Yes ||| No - id: b6abf882-a346-4a35-bf34-2067a3a6e9f3 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + Read the following two sentences. - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences + original_task: true + name: 'A/B choice randomized (choice order: B-A)' reference: '' - be59edae-2581-41e6-ac77-294704c93ce2: !Template + e629ff75-6fe5-4408-95ee-38f234f90843: !Template answer_choices: Yes ||| No - id: be59edae-2581-41e6-ac77-294704c93ce2 + id: e629ff75-6fe5-4408-95ee-38f234f90843 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -256,26 +240,26 @@ templates: metrics: - Accuracy original_task: false - name: Yes/No binary judgment for good sentences (choice order reversed) + name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' reference: '' - ef42acf8-084f-49ce-ad22-a5fc48a67d6d: !Template + f2062239-6ce0-49d3-92e5-8cf15fbf1bcc: !Template answer_choices: null - id: ef42acf8-084f-49ce-ad22-a5fc48a67d6d + id: f2062239-6ce0-49d3-92e5-8cf15fbf1bcc jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -283,11 +267,11 @@ templates: {% if shuffled_order == 0 %} - 1 + A {% else %} - 2 + B {% endif %}' metadata: !TemplateMetadata @@ -295,5 +279,21 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: random_choice_A_B + reference: '' + fd68288e-d2a3-453b-aa68-aab00a4f3211: !Template + answer_choices: Yes ||| No + id: fd68288e-d2a3-453b-aa68-aab00a4f3211 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' reference: '' diff --git a/promptsource/templates/blimp/only_npi_licensor_present/templates.yaml b/promptsource/templates/blimp/only_npi_licensor_present/templates.yaml index b030475fb..ed791a847 100644 --- a/promptsource/templates/blimp/only_npi_licensor_present/templates.yaml +++ b/promptsource/templates/blimp/only_npi_licensor_present/templates.yaml @@ -1,29 +1,52 @@ dataset: blimp subset: only_npi_licensor_present templates: - 0fe83ac3-77bb-4286-8e7c-0d424adef1ab: !Template - answer_choices: Yes ||| No - id: 0fe83ac3-77bb-4286-8e7c-0d424adef1ab - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + 1cdd74f2-ab69-4c48-8617-e56e98bae3cc: !Template + answer_choices: null + id: 1cdd74f2-ab69-4c48-8617-e56e98bae3cc + jinja: '{% set shuffled_order = [0, 1] | random %} - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences (choice order reversed) + original_task: true + name: random_choice_A_B_2 reference: '' - 17090f38-019d-46c0-9b26-47f40aee76c4: !Template + 23c8d5d2-88c9-46a9-9f62-ed98c285811d: !Template answer_choices: Yes ||| No - id: 17090f38-019d-46c0-9b26-47f40aee76c4 + id: 23c8d5d2-88c9-46a9-9f62-ed98c285811d jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -31,11 +54,11 @@ templates: metrics: - Accuracy original_task: false - name: Yes/No binary judgment for good sentences + name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' reference: '' - 4c89b8cb-1a0c-4e09-996f-e417e3acc286: !Template + 26f64782-b7db-4ce3-98d2-5ffe7f676ebb: !Template answer_choices: Yes ||| No - id: 4c89b8cb-1a0c-4e09-996f-e417e3acc286 + id: 26f64782-b7db-4ce3-98d2-5ffe7f676ebb jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -47,37 +70,38 @@ templates: metrics: - Accuracy original_task: false - name: Yes/No binary judgment for bad sentences (choice order reversed) + name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' reference: '' - 62e50096-d057-4c40-baa6-eda3b9a5a6b4: !Template - answer_choices: A ||| B - id: 62e50096-d057-4c40-baa6-eda3b9a5a6b4 + 352347bf-77cd-487c-b85d-35beb6be2aa6: !Template + answer_choices: null + id: 352347bf-77cd-487c-b85d-35beb6be2aa6 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| - {{ answer_choices[0] }} + {% if shuffled_order == 0 %} + + 1 {% else %} - {{ answer_choices[1] }} + 2 {% endif %}' metadata: !TemplateMetadata @@ -85,11 +109,11 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized (choice order reversed) + name: random_choice_1_2 reference: '' - 924c661d-6a2c-4fa7-bcb9-c28a913a191d: !Template + 3e4e757b-d419-451d-8842-d642610ac3d6: !Template answer_choices: Yes ||| No - id: 924c661d-6a2c-4fa7-bcb9-c28a913a191d + id: 3e4e757b-d419-451d-8842-d642610ac3d6 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -101,49 +125,11 @@ templates: metrics: - Accuracy original_task: false - name: Yes/No binary judgment for bad sentences - reference: '' - 98451869-5e77-423b-8e37-57f82450fa81: !Template - answer_choices: A ||| B - id: 98451869-5e77-423b-8e37-57f82450fa81 - jinja: '{% set shuffled_order = [0, 1] | random %} - - Read the following two sentences. - - {% if shuffled_order == 0 %} - - Sentence A: {{ sentence_good }} - - Sentence B: {{ sentence_bad }} - - {% else %} - - Sentence A: {{ sentence_bad }} - - Sentence B: {{ sentence_good }} - - {% endif %} - - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} - - {% endif %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: A/B choice randomized + name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' reference: '' - b0265345-5825-43e6-8e75-94b11028d938: !Template + 43c9b0be-8743-4a7f-9c88-abfd06beee8a: !Template answer_choices: null - id: b0265345-5825-43e6-8e75-94b11028d938 + id: 43c9b0be-8743-4a7f-9c88-abfd06beee8a jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -180,36 +166,51 @@ templates: original_task: true name: random_choice_1_2_2 reference: '' - bc8f2d54-6e71-4be9-8d2c-133254e56c55: !Template - answer_choices: null - id: bc8f2d54-6e71-4be9-8d2c-133254e56c55 + 6c93e5b3-38e2-454b-a56e-dd3d553ad092: !Template + answer_choices: Yes ||| No + id: 6c93e5b3-38e2-454b-a56e-dd3d553ad092 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' + reference: '' + 9bdbb8db-d390-4d2b-8503-24f08f74cf01: !Template + answer_choices: A ||| B + id: 9bdbb8db-d390-4d2b-8503-24f08f74cf01 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -217,26 +218,26 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: 'A/B choice randomized (choice order: A-B)' reference: '' - bfbb377d-86a6-4c2c-a603-d9d39a81ca81: !Template + c9eb9adf-5f0c-4e0b-99f3-cb8b7a508868: !Template answer_choices: null - id: bfbb377d-86a6-4c2c-a603-d9d39a81ca81 + id: c9eb9adf-5f0c-4e0b-99f3-cb8b7a508868 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -244,11 +245,11 @@ templates: {% if shuffled_order == 0 %} - 1 + A {% else %} - 2 + B {% endif %}' metadata: !TemplateMetadata @@ -256,38 +257,37 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: random_choice_A_B reference: '' - d7d2ac5d-101d-4ad2-ac5d-fa414ac7f502: !Template - answer_choices: null - id: d7d2ac5d-101d-4ad2-ac5d-fa414ac7f502 + dc1d3756-85a7-4165-819b-c9b198b7dbb7: !Template + answer_choices: A ||| B + id: dc1d3756-85a7-4165-819b-c9b198b7dbb7 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -295,5 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: 'A/B choice randomized (choice order: B-A)' reference: '' diff --git a/promptsource/templates/blimp/only_npi_scope/templates.yaml b/promptsource/templates/blimp/only_npi_scope/templates.yaml index 2c4b67341..b781c2a2a 100644 --- a/promptsource/templates/blimp/only_npi_scope/templates.yaml +++ b/promptsource/templates/blimp/only_npi_scope/templates.yaml @@ -1,41 +1,63 @@ dataset: blimp subset: only_npi_scope templates: - 004f0c42-5bb2-4695-9b63-90edf2264748: !Template - answer_choices: Yes ||| No - id: 004f0c42-5bb2-4695-9b63-90edf2264748 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + 00046d74-7f92-47d6-bee8-c1ef9f705227: !Template + answer_choices: A ||| B + id: 00046d74-7f92-47d6-bee8-c1ef9f705227 + jinja: '{% set shuffled_order = [0, 1] | random %} - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences + original_task: true + name: 'A/B choice randomized (choice order: A-B)' reference: '' - 105286db-c912-4bae-bfd0-1f5e1a560173: !Template + 1433ff94-edb2-462e-940d-333745f1287e: !Template answer_choices: Yes ||| No - id: 105286db-c912-4bae-bfd0-1f5e1a560173 + id: 1433ff94-edb2-462e-940d-333745f1287e jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: Yes/No binary judgment for good sentences (choice order reversed) + name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' reference: '' - 39ebdce9-997a-4c40-aeab-1e6dc35c1e3b: !Template + 14ba38d2-20f3-48c8-8993-5713888c8ea1: !Template answer_choices: A ||| B - id: 39ebdce9-997a-4c40-aeab-1e6dc35c1e3b + id: 14ba38d2-20f3-48c8-8993-5713888c8ea1 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -69,11 +91,11 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized (choice order reversed) + name: 'A/B choice randomized (choice order: B-A)' reference: '' - 3dceb71f-4e37-4ed9-a4f8-ad5c442fa7a8: !Template + 51ad5fec-f2d0-46a2-b0f3-899464f69e42: !Template answer_choices: null - id: 3dceb71f-4e37-4ed9-a4f8-ad5c442fa7a8 + id: 51ad5fec-f2d0-46a2-b0f3-899464f69e42 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer 1 or 2. @@ -110,47 +132,9 @@ templates: original_task: true name: random_choice_1_2 reference: '' - 5e04d59a-2491-4990-893c-eb8ad7d0c96a: !Template - answer_choices: A ||| B - id: 5e04d59a-2491-4990-893c-eb8ad7d0c96a - jinja: '{% set shuffled_order = [0, 1] | random %} - - Read the following two sentences. - - {% if shuffled_order == 0 %} - - Sentence A: {{ sentence_good }} - - Sentence B: {{ sentence_bad }} - - {% else %} - - Sentence A: {{ sentence_bad }} - - Sentence B: {{ sentence_good }} - - {% endif %} - - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} - - {% endif %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: A/B choice randomized - reference: '' - 8b450990-2efb-48e1-a903-4d5837951de2: !Template + 69b54363-2a35-4285-8b32-fadeed38d419: !Template answer_choices: null - id: 8b450990-2efb-48e1-a903-4d5837951de2 + id: 69b54363-2a35-4285-8b32-fadeed38d419 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer A or B. @@ -187,9 +171,9 @@ templates: original_task: true name: random_choice_A_B reference: '' - a4e614f0-7bb4-4723-86b4-3579f86a50a5: !Template + b3dee517-bada-4cca-80e4-d7793050f5ee: !Template answer_choices: null - id: a4e614f0-7bb4-4723-86b4-3579f86a50a5 + id: b3dee517-bada-4cca-80e4-d7793050f5ee jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer A or B. @@ -226,9 +210,57 @@ templates: original_task: true name: random_choice_A_B_2 reference: '' - c68b76db-1764-410d-b3cc-09c5cf0d3ebd: !Template + bdb12eb1-4ce4-4cc7-b9d9-646295f46d0c: !Template + answer_choices: Yes ||| No + id: bdb12eb1-4ce4-4cc7-b9d9-646295f46d0c + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' + reference: '' + c834463e-98c7-45d1-afe7-e9f0b61889b0: !Template + answer_choices: Yes ||| No + id: c834463e-98c7-45d1-afe7-e9f0b61889b0 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' + reference: '' + e8b39505-b8a5-4f3a-ba0a-ad8c60b9f173: !Template + answer_choices: Yes ||| No + id: e8b39505-b8a5-4f3a-ba0a-ad8c60b9f173 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' + reference: '' + fb0ce31d-bddb-4e3e-a4bd-b32b9f7918b1: !Template answer_choices: null - id: c68b76db-1764-410d-b3cc-09c5cf0d3ebd + id: fb0ce31d-bddb-4e3e-a4bd-b32b9f7918b1 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -265,35 +297,3 @@ templates: original_task: true name: random_choice_1_2_2 reference: '' - d903517e-dfec-43e1-880e-c8f4a48da0c0: !Template - answer_choices: Yes ||| No - id: d903517e-dfec-43e1-880e-c8f4a48da0c0 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences - reference: '' - e11d4e8e-8451-447e-be52-9848028c57d1: !Template - answer_choices: Yes ||| No - id: e11d4e8e-8451-447e-be52-9848028c57d1 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences (choice order reversed) - reference: '' diff --git a/promptsource/templates/blimp/passive_1/templates.yaml b/promptsource/templates/blimp/passive_1/templates.yaml index a75578a75..66427dc18 100644 --- a/promptsource/templates/blimp/passive_1/templates.yaml +++ b/promptsource/templates/blimp/passive_1/templates.yaml @@ -1,24 +1,24 @@ dataset: blimp subset: passive_1 templates: - 2f0cfc41-575a-4ac5-8d63-e5ac46b247de: !Template + 05508d69-7b11-4875-bd22-b722bc0a5fed: !Template answer_choices: null - id: 2f0cfc41-575a-4ac5-8d63-e5ac46b247de + id: 05508d69-7b11-4875-bd22-b722bc0a5fed jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -26,11 +26,11 @@ templates: {% if shuffled_order == 0 %} - A + 1 {% else %} - B + 2 {% endif %}' metadata: !TemplateMetadata @@ -38,11 +38,43 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: random_choice_1_2 + reference: '' + 099435ac-4216-4c5a-8c5c-47276c69ce2c: !Template + answer_choices: Yes ||| No + id: 099435ac-4216-4c5a-8c5c-47276c69ce2c + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' + reference: '' + 1a36c01f-575e-4226-88fb-e7c3deca5ea2: !Template + answer_choices: Yes ||| No + id: 1a36c01f-575e-4226-88fb-e7c3deca5ea2 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' reference: '' - 62971a37-542c-4c48-bc1b-d9f5d3801f25: !Template + 22941fb5-4e23-40e9-86df-8280bd2ca74f: !Template answer_choices: Yes ||| No - id: 62971a37-542c-4c48-bc1b-d9f5d3801f25 + id: 22941fb5-4e23-40e9-86df-8280bd2ca74f jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -54,11 +86,11 @@ templates: metrics: - Accuracy original_task: false - name: Yes/No binary judgment for good sentences (choice order reversed) + name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' reference: '' - 63b3fd0a-759e-44e2-91fe-ba7bf515813c: !Template + 5e6bee37-3698-4532-9201-7a766168a068: !Template answer_choices: A ||| B - id: 63b3fd0a-759e-44e2-91fe-ba7bf515813c + id: 5e6bee37-3698-4532-9201-7a766168a068 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -92,54 +124,37 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized + name: 'A/B choice randomized (choice order: A-B)' reference: '' - 711f6d10-413a-4745-8afd-a0e30588eec8: !Template - answer_choices: Yes ||| No - id: 711f6d10-413a-4745-8afd-a0e30588eec8 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences - reference: '' - 8d5ab8c2-948f-4a38-9fda-d6c9e142f147: !Template - answer_choices: null - id: 8d5ab8c2-948f-4a38-9fda-d6c9e142f147 + 859d74d6-b089-4cbd-a3bf-468bac5e49f2: !Template + answer_choices: A ||| B + id: 859d74d6-b089-4cbd-a3bf-468bac5e49f2 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -147,53 +162,38 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: 'A/B choice randomized (choice order: B-A)' reference: '' - 9be7454f-55aa-49b7-a30f-7c95d3311162: !Template - answer_choices: Yes ||| No - id: 9be7454f-55aa-49b7-a30f-7c95d3311162 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences (choice order reversed) - reference: '' - a779ca82-98c2-4bae-a29f-9bf61f402166: !Template - answer_choices: A ||| B - id: a779ca82-98c2-4bae-a29f-9bf61f402166 + 90d95bbb-4370-42b6-b331-8153ddf0b629: !Template + answer_choices: null + id: 90d95bbb-4370-42b6-b331-8153ddf0b629 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| - {{ answer_choices[0] }} + {% if shuffled_order == 0 %} + + A {% else %} - {{ answer_choices[1] }} + B {% endif %}' metadata: !TemplateMetadata @@ -201,14 +201,14 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized (choice order reversed) + name: random_choice_A_B_2 reference: '' - c7ade213-fc39-4881-b2f5-2bf240851c52: !Template + 9c0e4e8f-aa33-4050-85f1-a20216484922: !Template answer_choices: null - id: c7ade213-fc39-4881-b2f5-2bf240851c52 + id: 9c0e4e8f-aa33-4050-85f1-a20216484922 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -240,11 +240,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: random_choice_1_2_2 reference: '' - ca4a66a8-ccbb-422b-b1d3-dc1cfef1c560: !Template + bcfb3886-95fa-4bbc-9b77-20fe4a3002bb: !Template answer_choices: Yes ||| No - id: ca4a66a8-ccbb-422b-b1d3-dc1cfef1c560 + id: bcfb3886-95fa-4bbc-9b77-20fe4a3002bb jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -256,26 +256,26 @@ templates: metrics: - Accuracy original_task: false - name: Yes/No binary judgment for bad sentences + name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' reference: '' - fa74b05a-f043-4703-90ca-b55f7857450a: !Template + e264fa32-baa8-49fe-8ea4-2ec2d028768d: !Template answer_choices: null - id: fa74b05a-f043-4703-90ca-b55f7857450a + id: e264fa32-baa8-49fe-8ea4-2ec2d028768d jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -283,11 +283,11 @@ templates: {% if shuffled_order == 0 %} - 1 + A {% else %} - 2 + B {% endif %}' metadata: !TemplateMetadata @@ -295,5 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: random_choice_A_B reference: '' diff --git a/promptsource/templates/blimp/passive_2/templates.yaml b/promptsource/templates/blimp/passive_2/templates.yaml index f4cacc498..ef8d2cd67 100644 --- a/promptsource/templates/blimp/passive_2/templates.yaml +++ b/promptsource/templates/blimp/passive_2/templates.yaml @@ -1,9 +1,25 @@ dataset: blimp subset: passive_2 templates: - 0db23227-68fd-45ba-9657-a780db36aa2a: !Template + 06850205-9e5d-418a-8669-c7458c270e4a: !Template + answer_choices: Yes ||| No + id: 06850205-9e5d-418a-8669-c7458c270e4a + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' + reference: '' + 1396a0a6-c44c-4199-bc35-0a0ccb0b93dd: !Template answer_choices: A ||| B - id: 0db23227-68fd-45ba-9657-a780db36aa2a + id: 1396a0a6-c44c-4199-bc35-0a0ccb0b93dd jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -22,7 +38,7 @@ templates: {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order + Which one is a better sentence of English, B or A? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -37,54 +53,47 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized + name: 'A/B choice randomized (choice order: B-A)' reference: '' - 2b8e7e36-b410-4615-a821-43b9e24d301d: !Template - answer_choices: null - id: 2b8e7e36-b410-4615-a821-43b9e24d301d - jinja: '{% set shuffled_order = [0, 1] | random %} - - Between the two sentences, which one is grammatical? Please answer A or B. - - {% if shuffled_order == 0 %} - - A: {{ sentence_good }} - - B: {{ sentence_bad }} - - {% else %} - - A: {{ sentence_bad }} - - B: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - A - - {% else %} + 235a154a-caef-4baa-92cb-074450405264: !Template + answer_choices: Yes ||| No + id: 235a154a-caef-4baa-92cb-074450405264 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - B + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' + reference: '' + 276507d3-82cb-4cdc-9991-ff8ceeb47af2: !Template + answer_choices: Yes ||| No + id: 276507d3-82cb-4cdc-9991-ff8ceeb47af2 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - {% endif %}' + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: random_choice_A_B_2 + original_task: false + name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' reference: '' - 4ac2289f-b34c-4795-8541-f5d6d8688950: !Template + 362fd9f3-e565-4145-831c-8abb42a2253f: !Template answer_choices: Yes ||| No - id: 4ac2289f-b34c-4795-8541-f5d6d8688950 + id: 362fd9f3-e565-4145-831c-8abb42a2253f jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -92,11 +101,11 @@ templates: metrics: - Accuracy original_task: false - name: Yes/No binary judgment for good sentences + name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' reference: '' - 6a5ae41a-3d8f-4733-b158-734c1d0538b7: !Template + 49a67e35-5c92-43fc-b94a-bee2b1b5af29: !Template answer_choices: null - id: 6a5ae41a-3d8f-4733-b158-734c1d0538b7 + id: 49a67e35-5c92-43fc-b94a-bee2b1b5af29 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer A or B. @@ -133,12 +142,12 @@ templates: original_task: true name: random_choice_A_B reference: '' - 72e7d5c3-aca0-4851-a0fd-e0c275c67c81: !Template + 87c52961-6c07-4499-965c-ec592a9de057: !Template answer_choices: null - id: 72e7d5c3-aca0-4851-a0fd-e0c275c67c81 + id: 87c52961-6c07-4499-965c-ec592a9de057 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -170,46 +179,53 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: random_choice_1_2_2 reference: '' - 8605bea0-1e0c-478c-8527-76fbc3dbb894: !Template - answer_choices: Yes ||| No - id: 8605bea0-1e0c-478c-8527-76fbc3dbb894 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + a31d9888-28d2-43f3-b781-033c28d16025: !Template + answer_choices: null + id: a31d9888-28d2-43f3-b781-033c28d16025 + jinja: '{% set shuffled_order = [0, 1] | random %} - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences (choice order reversed) - reference: '' - 8df0b50c-9efd-4dd2-b660-6a912d1d9f43: !Template - answer_choices: Yes ||| No - id: 8df0b50c-9efd-4dd2-b660-6a912d1d9f43 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + Between the two sentences, which one is grammatical? Please answer A or B. - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences + original_task: true + name: random_choice_A_B_2 reference: '' - 9ef44939-488f-4a2a-9670-fd85750584f7: !Template + ade2a82f-aab8-4b21-9975-22dc3cfde035: !Template answer_choices: null - id: 9ef44939-488f-4a2a-9670-fd85750584f7 + id: ade2a82f-aab8-4b21-9975-22dc3cfde035 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -241,27 +257,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 - reference: '' - e84d0c9e-ba07-464f-9ffb-b762864a5db6: !Template - answer_choices: Yes ||| No - id: e84d0c9e-ba07-464f-9ffb-b762864a5db6 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences (choice order reversed) + name: random_choice_1_2 reference: '' - fdef417a-2746-4e96-ab73-cc6fcd748d57: !Template + f8abe76f-8b7f-4fb8-b8b3-3d6e524ec006: !Template answer_choices: A ||| B - id: fdef417a-2746-4e96-ab73-cc6fcd748d57 + id: f8abe76f-8b7f-4fb8-b8b3-3d6e524ec006 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -280,7 +280,7 @@ templates: {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order + Which one is a better sentence of English, A or B? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -295,5 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized (choice order reversed) + name: 'A/B choice randomized (choice order: A-B)' reference: '' diff --git a/promptsource/templates/blimp/principle_A_c_command/templates.yaml b/promptsource/templates/blimp/principle_A_c_command/templates.yaml index debbc30e3..bdfffd8f7 100644 --- a/promptsource/templates/blimp/principle_A_c_command/templates.yaml +++ b/promptsource/templates/blimp/principle_A_c_command/templates.yaml @@ -1,47 +1,9 @@ dataset: blimp subset: principle_A_c_command templates: - 2baf720f-f276-4453-bc65-aa1243db1628: !Template - answer_choices: A ||| B - id: 2baf720f-f276-4453-bc65-aa1243db1628 - jinja: '{% set shuffled_order = [0, 1] | random %} - - Read the following two sentences. - - {% if shuffled_order == 0 %} - - Sentence A: {{ sentence_good }} - - Sentence B: {{ sentence_bad }} - - {% else %} - - Sentence A: {{ sentence_bad }} - - Sentence B: {{ sentence_good }} - - {% endif %} - - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} - - {% endif %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: A/B choice randomized - reference: '' - 3ccc7b98-a1f0-40d0-887f-6b8b4d02be85: !Template + 3a8a234f-6810-4066-85e9-0719796eecca: !Template answer_choices: Yes ||| No - id: 3ccc7b98-a1f0-40d0-887f-6b8b4d02be85 + id: 3a8a234f-6810-4066-85e9-0719796eecca jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -53,26 +15,26 @@ templates: metrics: - Accuracy original_task: false - name: Yes/No binary judgment for bad sentences (choice order reversed) + name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' reference: '' - 52726eee-092b-45c1-92c7-5846729ad713: !Template + 41e0a368-eb19-436c-87b3-526b5c97fd62: !Template answer_choices: null - id: 52726eee-092b-45c1-92c7-5846729ad713 + id: 41e0a368-eb19-436c-87b3-526b5c97fd62 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -80,11 +42,11 @@ templates: {% if shuffled_order == 0 %} - A + 1 {% else %} - B + 2 {% endif %}' metadata: !TemplateMetadata @@ -92,43 +54,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B - reference: '' - 5b929665-8a28-404d-b199-badc6d460dfc: !Template - answer_choices: Yes ||| No - id: 5b929665-8a28-404d-b199-badc6d460dfc - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences - reference: '' - 7170e0c2-e190-4749-acc9-8728a24e2ead: !Template - answer_choices: Yes ||| No - id: 7170e0c2-e190-4749-acc9-8728a24e2ead - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences (choice order reversed) + name: random_choice_1_2 reference: '' - 81723db6-ba6e-44c8-9ac1-bcdf85e6fd4d: !Template + 45a8509c-86c2-4f82-8bc1-bb029b23e40e: !Template answer_choices: A ||| B - id: 81723db6-ba6e-44c8-9ac1-bcdf85e6fd4d + id: 45a8509c-86c2-4f82-8bc1-bb029b23e40e jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -162,11 +92,11 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized (choice order reversed) + name: 'A/B choice randomized (choice order: B-A)' reference: '' - 84c688a8-d189-472f-bce2-3086b989f9a7: !Template + 7a67aeb5-50b3-49cc-b3a8-e149135e9b55: !Template answer_choices: null - id: 84c688a8-d189-472f-bce2-3086b989f9a7 + id: 7a67aeb5-50b3-49cc-b3a8-e149135e9b55 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -203,13 +133,13 @@ templates: original_task: true name: random_choice_1_2_2 reference: '' - 8d7af12a-6b0a-47cf-aafa-09d9f9c4adf2: !Template + 7cf054e0-5fe2-4ff6-ab66-17d8b72ece00: !Template answer_choices: Yes ||| No - id: 8d7af12a-6b0a-47cf-aafa-09d9f9c4adf2 + id: 7cf054e0-5fe2-4ff6-ab66-17d8b72ece00 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -217,26 +147,80 @@ templates: metrics: - Accuracy original_task: false - name: Yes/No binary judgment for good sentences + name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' + reference: '' + 87a1a3e4-753f-429c-9712-09eee687492a: !Template + answer_choices: A ||| B + id: 87a1a3e4-753f-429c-9712-09eee687492a + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice randomized (choice order: A-B)' + reference: '' + 87a371fe-e4a4-43a1-a2ee-0d6d8756c450: !Template + answer_choices: Yes ||| No + id: 87a371fe-e4a4-43a1-a2ee-0d6d8756c450 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' reference: '' - 9c5d4d2c-89fe-4536-98a6-40d6d406ef0b: !Template + 8aab460d-3ae1-4d12-afb7-121ad025f65f: !Template answer_choices: null - id: 9c5d4d2c-89fe-4536-98a6-40d6d406ef0b + id: 8aab460d-3ae1-4d12-afb7-121ad025f65f jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -244,11 +228,11 @@ templates: {% if shuffled_order == 0 %} - 1 + A {% else %} - 2 + B {% endif %}' metadata: !TemplateMetadata @@ -256,11 +240,27 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: random_choice_A_B + reference: '' + df9d8964-6158-4fe7-828a-15249921878c: !Template + answer_choices: Yes ||| No + id: df9d8964-6158-4fe7-828a-15249921878c + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' reference: '' - cf193f1f-4c5c-4472-b5c3-bbdd0840c41c: !Template + e6b34bec-c16a-471a-975e-ebfcda39f997: !Template answer_choices: null - id: cf193f1f-4c5c-4472-b5c3-bbdd0840c41c + id: e6b34bec-c16a-471a-975e-ebfcda39f997 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer A or B. diff --git a/promptsource/templates/blimp/principle_A_case_1/templates.yaml b/promptsource/templates/blimp/principle_A_case_1/templates.yaml index aaaaeb277..6a4086f4f 100644 --- a/promptsource/templates/blimp/principle_A_case_1/templates.yaml +++ b/promptsource/templates/blimp/principle_A_case_1/templates.yaml @@ -1,24 +1,24 @@ dataset: blimp subset: principle_A_case_1 templates: - 015dcbc0-3c03-4567-954e-e827cb378af5: !Template + 006e75f2-cc80-4859-b1ae-11e44c9943d3: !Template answer_choices: null - id: 015dcbc0-3c03-4567-954e-e827cb378af5 + id: 006e75f2-cc80-4859-b1ae-11e44c9943d3 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -26,11 +26,11 @@ templates: {% if shuffled_order == 0 %} - A + 1 {% else %} - B + 2 {% endif %}' metadata: !TemplateMetadata @@ -38,15 +38,15 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: random_choice_1_2 reference: '' - 36eb9af7-58d0-4015-afdd-7368bed1c976: !Template + 3a0dfa3b-d275-48dd-9ae6-781135300ec2: !Template answer_choices: Yes ||| No - id: 36eb9af7-58d0-4015-afdd-7368bed1c976 + id: 3a0dfa3b-d275-48dd-9ae6-781135300ec2 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata @@ -54,38 +54,37 @@ templates: metrics: - Accuracy original_task: false - name: Yes/No binary judgment for bad sentences + name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' reference: '' - 45e98aed-70ef-40b5-b632-3bdda7691c7e: !Template - answer_choices: null - id: 45e98aed-70ef-40b5-b632-3bdda7691c7e + 53dc1cdd-da59-4e23-bfd6-87482f76c20d: !Template + answer_choices: A ||| B + id: 53dc1cdd-da59-4e23-bfd6-87482f76c20d jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -93,26 +92,42 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: 'A/B choice randomized (choice order: B-A)' reference: '' - 6720534d-1a15-49e9-b7f1-63a6afe34ea0: !Template + 699f6634-2536-40c5-94a8-385697c32063: !Template + answer_choices: Yes ||| No + id: 699f6634-2536-40c5-94a8-385697c32063 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' + reference: '' + 71efac31-53cf-4a96-9922-5d36138fe6de: !Template answer_choices: null - id: 6720534d-1a15-49e9-b7f1-63a6afe34ea0 + id: 71efac31-53cf-4a96-9922-5d36138fe6de jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -120,11 +135,11 @@ templates: {% if shuffled_order == 0 %} - 1 + A {% else %} - 2 + B {% endif %}' metadata: !TemplateMetadata @@ -132,11 +147,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: random_choice_A_B reference: '' - 733c3230-e0eb-479d-bfd6-bfc60651076d: !Template + afcc2ba1-4b8d-4718-bfc3-6a442b4d319a: !Template answer_choices: A ||| B - id: 733c3230-e0eb-479d-bfd6-bfc60651076d + id: afcc2ba1-4b8d-4718-bfc3-6a442b4d319a jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -170,53 +185,70 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized + name: 'A/B choice randomized (choice order: A-B)' reference: '' - 998c717c-041a-4a39-890f-00dd33d923dc: !Template + b7b20894-3825-497f-b21c-9de10513a64b: !Template answer_choices: Yes ||| No - id: 998c717c-041a-4a39-890f-00dd33d923dc + id: b7b20894-3825-497f-b21c-9de10513a64b jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: Yes/No binary judgment for bad sentences (choice order reversed) + name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' reference: '' - 9cfdebd3-c056-4a49-b479-86c259a4b349: !Template - answer_choices: A ||| B - id: 9cfdebd3-c056-4a49-b479-86c259a4b349 + e023d42b-d5a9-42ae-9153-3fa06a8a32a9: !Template + answer_choices: Yes ||| No + id: e023d42b-d5a9-42ae-9153-3fa06a8a32a9 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' + reference: '' + ecab2028-3d20-4174-be3c-ef6ba6ea813c: !Template + answer_choices: null + id: ecab2028-3d20-4174-be3c-ef6ba6ea813c jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| - {{ answer_choices[0] }} + {% if shuffled_order == 0 %} + + A {% else %} - {{ answer_choices[1] }} + B {% endif %}' metadata: !TemplateMetadata @@ -224,58 +256,26 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized (choice order reversed) - reference: '' - c42a7b10-1ec8-4fec-aeac-c3803047f2df: !Template - answer_choices: Yes ||| No - id: c42a7b10-1ec8-4fec-aeac-c3803047f2df - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences (choice order reversed) - reference: '' - f8337f64-b45d-4e1a-8523-f6ec734e942d: !Template - answer_choices: Yes ||| No - id: f8337f64-b45d-4e1a-8523-f6ec734e942d - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences + name: random_choice_A_B_2 reference: '' - fc4314a3-ec10-4d15-8d66-fd6b8b02b7b9: !Template + f1ccd522-fa41-446c-b60e-372364b44274: !Template answer_choices: null - id: fc4314a3-ec10-4d15-8d66-fd6b8b02b7b9 + id: f1ccd522-fa41-446c-b60e-372364b44274 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -283,11 +283,11 @@ templates: {% if shuffled_order == 0 %} - A + 1 {% else %} - B + 2 {% endif %}' metadata: !TemplateMetadata @@ -295,5 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: random_choice_1_2_2 reference: '' diff --git a/promptsource/templates/blimp/principle_A_case_2/templates.yaml b/promptsource/templates/blimp/principle_A_case_2/templates.yaml index 8a953353f..2c868a1b1 100644 --- a/promptsource/templates/blimp/principle_A_case_2/templates.yaml +++ b/promptsource/templates/blimp/principle_A_case_2/templates.yaml @@ -1,64 +1,9 @@ dataset: blimp subset: principle_A_case_2 templates: - 0e06f97d-bd62-4a61-a8b5-2720938f0c8b: !Template - answer_choices: Yes ||| No - id: 0e06f97d-bd62-4a61-a8b5-2720938f0c8b - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences (choice order reversed) - reference: '' - 36af5545-6402-467f-9910-4afaa4fe8e83: !Template - answer_choices: null - id: 36af5545-6402-467f-9910-4afaa4fe8e83 - jinja: '{% set shuffled_order = [0, 1] | random %} - - Which one of the following sentences is grammatical? Please answer 1 or 2. - - {% if shuffled_order == 0 %} - - 1: {{ sentence_good }} - - 2: {{ sentence_bad }} - - {% else %} - - 1: {{ sentence_bad }} - - 2: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - 1 - - {% else %} - - 2 - - {% endif %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: random_choice_1_2 - reference: '' - 5f9bb974-505c-4cbd-91dd-aaf624cc9155: !Template + 22577b88-67fd-4fa3-a89a-e5bb1da31047: !Template answer_choices: A ||| B - id: 5f9bb974-505c-4cbd-91dd-aaf624cc9155 + id: 22577b88-67fd-4fa3-a89a-e5bb1da31047 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -77,7 +22,7 @@ templates: {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order + Which one is a better sentence of English, A or B? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -92,27 +37,27 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized (choice order reversed) + name: 'A/B choice randomized (choice order: A-B)' reference: '' - 645d1908-7f34-4d1b-899c-a7ba3defcc17: !Template + 5a8038d7-d922-4cf1-9447-65f732c81ccf: !Template answer_choices: Yes ||| No - id: 645d1908-7f34-4d1b-899c-a7ba3defcc17 + id: 5a8038d7-d922-4cf1-9447-65f732c81ccf jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: Yes/No binary judgment for good sentences (choice order reversed) + name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' reference: '' - 7f8f8557-5a60-4901-a6d1-18b7a9f543fe: !Template + 633080ad-aa2c-409a-8912-edf3fd02f81f: !Template answer_choices: null - id: 7f8f8557-5a60-4901-a6d1-18b7a9f543fe + id: 633080ad-aa2c-409a-8912-edf3fd02f81f jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -149,24 +94,40 @@ templates: original_task: true name: random_choice_1_2_2 reference: '' - 889961b0-cf52-4659-ad58-5372883bb19f: !Template + 80231c1a-b0a4-4d6c-88b4-b81d708c8442: !Template + answer_choices: Yes ||| No + id: 80231c1a-b0a4-4d6c-88b4-b81d708c8442 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' + reference: '' + 8a1b6b9a-20f1-4450-a848-3838c45fa7e3: !Template answer_choices: null - id: 889961b0-cf52-4659-ad58-5372883bb19f + id: 8a1b6b9a-20f1-4450-a848-3838c45fa7e3 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -174,11 +135,11 @@ templates: {% if shuffled_order == 0 %} - A + 1 {% else %} - B + 2 {% endif %}' metadata: !TemplateMetadata @@ -186,11 +147,43 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: random_choice_1_2 reference: '' - 8cdda726-0e4c-44f0-a986-793e9bd5d8d2: !Template + 9f8b70a6-28ac-4ee7-adf4-6a0d99ab7f61: !Template + answer_choices: Yes ||| No + id: 9f8b70a6-28ac-4ee7-adf4-6a0d99ab7f61 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' + reference: '' + afbc2fa9-0571-4d74-a9a2-a029650c604a: !Template + answer_choices: Yes ||| No + id: afbc2fa9-0571-4d74-a9a2-a029650c604a + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' + reference: '' + bd7eb31a-0b27-48b5-a97e-a060c93c57bd: !Template answer_choices: null - id: 8cdda726-0e4c-44f0-a986-793e9bd5d8d2 + id: bd7eb31a-0b27-48b5-a97e-a060c93c57bd jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer A or B. @@ -227,25 +220,9 @@ templates: original_task: true name: random_choice_A_B reference: '' - 99743879-4158-4cc2-a65a-30b88ae9be63: !Template - answer_choices: Yes ||| No - id: 99743879-4158-4cc2-a65a-30b88ae9be63 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences - reference: '' - b62169dc-11f0-409d-aff7-c4e1c18e5f5f: !Template + d2df2724-697e-453a-9a9d-d3d149bab653: !Template answer_choices: A ||| B - id: b62169dc-11f0-409d-aff7-c4e1c18e5f5f + id: d2df2724-697e-453a-9a9d-d3d149bab653 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -264,7 +241,7 @@ templates: {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order + Which one is a better sentence of English, B or A? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -279,21 +256,44 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized + name: 'A/B choice randomized (choice order: B-A)' reference: '' - e204ff36-aed4-4b9c-9f22-2f759e15dfb7: !Template - answer_choices: Yes ||| No - id: e204ff36-aed4-4b9c-9f22-2f759e15dfb7 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + d497385a-b15a-4677-a92b-bd0c9f2e4434: !Template + answer_choices: null + id: d497385a-b15a-4677-a92b-bd0c9f2e4434 + jinja: '{% set shuffled_order = [0, 1] | random %} - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences + original_task: true + name: random_choice_A_B_2 reference: '' diff --git a/promptsource/templates/blimp/principle_A_domain_1/templates.yaml b/promptsource/templates/blimp/principle_A_domain_1/templates.yaml index 7b49d7743..a840e53b1 100644 --- a/promptsource/templates/blimp/principle_A_domain_1/templates.yaml +++ b/promptsource/templates/blimp/principle_A_domain_1/templates.yaml @@ -1,12 +1,28 @@ dataset: blimp subset: principle_A_domain_1 templates: - 1eac62d1-f9c0-4937-8161-cdedbfd90b60: !Template + 19cb8530-a493-4f6e-afd4-96658e47480f: !Template + answer_choices: Yes ||| No + id: 19cb8530-a493-4f6e-afd4-96658e47480f + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' + reference: '' + 83d5c36b-4acd-4db9-9bd1-aa8954c571fa: !Template answer_choices: null - id: 1eac62d1-f9c0-4937-8161-cdedbfd90b60 + id: 83d5c36b-4acd-4db9-9bd1-aa8954c571fa jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -38,54 +54,53 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: random_choice_A_B_2 reference: '' - 5b2541e0-4d4f-4dc1-b551-5484226ff639: !Template + 850265b6-8463-47dc-b019-a01a033fd6d6: !Template answer_choices: Yes ||| No - id: 5b2541e0-4d4f-4dc1-b551-5484226ff639 + id: 850265b6-8463-47dc-b019-a01a033fd6d6 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: Yes/No binary judgment for good sentences (choice order reversed) + name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' reference: '' - 6b667e46-ba6b-472a-beed-4ea311ce13ea: !Template - answer_choices: null - id: 6b667e46-ba6b-472a-beed-4ea311ce13ea + 96b0ff1d-ba98-4ccb-8c27-472d93e06eb9: !Template + answer_choices: A ||| B + id: 96b0ff1d-ba98-4ccb-8c27-472d93e06eb9 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -93,42 +108,26 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 - reference: '' - 8d398353-b666-4065-8e94-5e4371097553: !Template - answer_choices: Yes ||| No - id: 8d398353-b666-4065-8e94-5e4371097553 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences (choice order reversed) + name: 'A/B choice randomized (choice order: B-A)' reference: '' - 9534d724-9d22-4ddf-ad3e-4e46f23e0749: !Template + 97939c02-5c5f-40b0-b22c-0fbc430c5730: !Template answer_choices: null - id: 9534d724-9d22-4ddf-ad3e-4e46f23e0749 + id: 97939c02-5c5f-40b0-b22c-0fbc430c5730 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -136,11 +135,11 @@ templates: {% if shuffled_order == 0 %} - 1 + A {% else %} - 2 + B {% endif %}' metadata: !TemplateMetadata @@ -148,27 +147,27 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: random_choice_A_B reference: '' - 9c95c009-2d1e-45a4-b7c5-37002b708ddd: !Template + be1cdccf-c158-49a5-bdc0-c34d5973eafe: !Template answer_choices: Yes ||| No - id: 9c95c009-2d1e-45a4-b7c5-37002b708ddd + id: be1cdccf-c158-49a5-bdc0-c34d5973eafe jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: Yes/No binary judgment for bad sentences + name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' reference: '' - b4ac6f8b-a749-47e7-9296-43677a4f0ab9: !Template + e0518dfb-31b3-4bb4-953d-877ae9394b6e: !Template answer_choices: Yes ||| No - id: b4ac6f8b-a749-47e7-9296-43677a4f0ab9 + id: e0518dfb-31b3-4bb4-953d-877ae9394b6e jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -180,14 +179,14 @@ templates: metrics: - Accuracy original_task: false - name: Yes/No binary judgment for good sentences + name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' reference: '' - c1dc8b28-d454-4a66-b377-19d132d7d989: !Template + ed330c7f-4450-4abb-9344-56e7a53d610a: !Template answer_choices: null - id: c1dc8b28-d454-4a66-b377-19d132d7d989 + id: ed330c7f-4450-4abb-9344-56e7a53d610a jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -219,37 +218,38 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: random_choice_1_2 reference: '' - c4083af3-6f77-4007-ac92-094ff365c8cf: !Template - answer_choices: A ||| B - id: c4083af3-6f77-4007-ac92-094ff365c8cf + f1f5e14e-d56b-41b2-9ab8-99228e95e213: !Template + answer_choices: null + id: f1f5e14e-d56b-41b2-9ab8-99228e95e213 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| - {{ answer_choices[0] }} + {% if shuffled_order == 0 %} + + 1 {% else %} - {{ answer_choices[1] }} + 2 {% endif %}' metadata: !TemplateMetadata @@ -257,11 +257,11 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized + name: random_choice_1_2_2 reference: '' - e13ab047-478e-4990-bc57-7db558d026a0: !Template + fa68b1c8-becf-49e3-a504-e3f6acc9a12e: !Template answer_choices: A ||| B - id: e13ab047-478e-4990-bc57-7db558d026a0 + id: fa68b1c8-becf-49e3-a504-e3f6acc9a12e jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -280,7 +280,7 @@ templates: {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order + Which one is a better sentence of English, A or B? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -295,5 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized (choice order reversed) + name: 'A/B choice randomized (choice order: A-B)' reference: '' diff --git a/promptsource/templates/blimp/principle_A_domain_2/templates.yaml b/promptsource/templates/blimp/principle_A_domain_2/templates.yaml index 3f5911adc..2f6c2f7be 100644 --- a/promptsource/templates/blimp/principle_A_domain_2/templates.yaml +++ b/promptsource/templates/blimp/principle_A_domain_2/templates.yaml @@ -1,9 +1,25 @@ dataset: blimp subset: principle_A_domain_2 templates: - 11974cf5-c168-4764-8274-85fa22cf73e4: !Template + 1f88e383-0be4-48e8-918b-0fc3211c27c5: !Template + answer_choices: Yes ||| No + id: 1f88e383-0be4-48e8-918b-0fc3211c27c5 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' + reference: '' + 22ac623e-f682-430b-a8ce-bf61b0ccbc03: !Template answer_choices: A ||| B - id: 11974cf5-c168-4764-8274-85fa22cf73e4 + id: 22ac623e-f682-430b-a8ce-bf61b0ccbc03 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -37,11 +53,50 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized (choice order reversed) + name: 'A/B choice randomized (choice order: B-A)' + reference: '' + 3a9794d2-8772-4a62-bb55-ea1f80ccdabb: !Template + answer_choices: null + id: 3a9794d2-8772-4a62-bb55-ea1f80ccdabb + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B reference: '' - 219461e6-ef46-443c-870e-8ec5bf400090: !Template + 41bc5343-54e9-4ae2-949d-810f5702c903: !Template answer_choices: null - id: 219461e6-ef46-443c-870e-8ec5bf400090 + id: 41bc5343-54e9-4ae2-949d-810f5702c903 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer 1 or 2. @@ -78,9 +133,9 @@ templates: original_task: true name: random_choice_1_2 reference: '' - 27659ea3-841d-4d92-a146-784bc8d1cdb4: !Template + a6903eb8-a4a3-479d-83cf-34c69f35bde2: !Template answer_choices: null - id: 27659ea3-841d-4d92-a146-784bc8d1cdb4 + id: a6903eb8-a4a3-479d-83cf-34c69f35bde2 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer A or B. @@ -117,9 +172,9 @@ templates: original_task: true name: random_choice_A_B_2 reference: '' - 2d7eb2d3-780b-4159-aea2-2126a0e4eb9e: !Template + e096e078-8553-4183-8cfd-ae1b15dce783: !Template answer_choices: null - id: 2d7eb2d3-780b-4159-aea2-2126a0e4eb9e + id: e096e078-8553-4183-8cfd-ae1b15dce783 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -156,9 +211,9 @@ templates: original_task: true name: random_choice_1_2_2 reference: '' - 33816ce2-a5e8-4c69-85ab-08ba1bcaa743: !Template + e43587f3-9cd6-4c70-935b-5453e7620967: !Template answer_choices: Yes ||| No - id: 33816ce2-a5e8-4c69-85ab-08ba1bcaa743 + id: e43587f3-9cd6-4c70-935b-5453e7620967 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -170,54 +225,37 @@ templates: metrics: - Accuracy original_task: false - name: Yes/No binary judgment for good sentences - reference: '' - 4239e2b0-a994-409f-a01a-505765885096: !Template - answer_choices: Yes ||| No - id: 4239e2b0-a994-409f-a01a-505765885096 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences + name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' reference: '' - 45b30932-072c-48b4-9134-93ff1ca0656f: !Template - answer_choices: null - id: 45b30932-072c-48b4-9134-93ff1ca0656f + edfcc4b1-5de9-4d28-8420-5dfbe8dff874: !Template + answer_choices: A ||| B + id: edfcc4b1-5de9-4d28-8420-5dfbe8dff874 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -225,11 +263,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: 'A/B choice randomized (choice order: A-B)' reference: '' - 8b4608d9-5243-4696-b2bb-c5ee0c130aa7: !Template + efde4ccd-524a-435c-a037-4e9fcc938e5a: !Template answer_choices: Yes ||| No - id: 8b4608d9-5243-4696-b2bb-c5ee0c130aa7 + id: efde4ccd-524a-435c-a037-4e9fcc938e5a jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -241,11 +279,11 @@ templates: metrics: - Accuracy original_task: false - name: Yes/No binary judgment for good sentences (choice order reversed) + name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' reference: '' - a397a1a9-9c20-429a-9c9f-bd423e004332: !Template + f30a0040-03da-4750-9a37-16ed02ee1fa2: !Template answer_choices: Yes ||| No - id: a397a1a9-9c20-429a-9c9f-bd423e004332 + id: f30a0040-03da-4750-9a37-16ed02ee1fa2 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -257,43 +295,5 @@ templates: metrics: - Accuracy original_task: false - name: Yes/No binary judgment for bad sentences (choice order reversed) - reference: '' - d463b428-ebd8-4cb6-a70e-d4316a829a0e: !Template - answer_choices: A ||| B - id: d463b428-ebd8-4cb6-a70e-d4316a829a0e - jinja: '{% set shuffled_order = [0, 1] | random %} - - Read the following two sentences. - - {% if shuffled_order == 0 %} - - Sentence A: {{ sentence_good }} - - Sentence B: {{ sentence_bad }} - - {% else %} - - Sentence A: {{ sentence_bad }} - - Sentence B: {{ sentence_good }} - - {% endif %} - - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} - - {% endif %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: A/B choice randomized + name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' reference: '' diff --git a/promptsource/templates/blimp/principle_A_domain_3/templates.yaml b/promptsource/templates/blimp/principle_A_domain_3/templates.yaml index d903b1220..f20f307ba 100644 --- a/promptsource/templates/blimp/principle_A_domain_3/templates.yaml +++ b/promptsource/templates/blimp/principle_A_domain_3/templates.yaml @@ -1,82 +1,12 @@ dataset: blimp subset: principle_A_domain_3 templates: - 017c8bef-e473-4c98-8e72-e27f84c0653f: !Template - answer_choices: Yes ||| No - id: 017c8bef-e473-4c98-8e72-e27f84c0653f - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences (choice order reversed) - reference: '' - 38bbc0c5-a9a3-4d8b-938e-bd66878d780a: !Template - answer_choices: Yes ||| No - id: 38bbc0c5-a9a3-4d8b-938e-bd66878d780a - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences - reference: '' - 632cb9c1-23a4-4744-b3c4-c606e6ee36e1: !Template - answer_choices: A ||| B - id: 632cb9c1-23a4-4744-b3c4-c606e6ee36e1 - jinja: '{% set shuffled_order = [0, 1] | random %} - - Read the following two sentences. - - {% if shuffled_order == 0 %} - - Sentence A: {{ sentence_good }} - - Sentence B: {{ sentence_bad }} - - {% else %} - - Sentence A: {{ sentence_bad }} - - Sentence B: {{ sentence_good }} - - {% endif %} - - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} - - {% endif %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: A/B choice randomized (choice order reversed) - reference: '' - 6be19da9-f28c-4620-9215-324a42f9d6ba: !Template + 25577715-21c2-482c-b8a6-8e4e14af5dd2: !Template answer_choices: null - id: 6be19da9-f28c-4620-9215-324a42f9d6ba + id: 25577715-21c2-482c-b8a6-8e4e14af5dd2 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -108,11 +38,27 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: random_choice_1_2 + reference: '' + 4a98eef9-e316-44f0-b8e5-ecc7d7adcf54: !Template + answer_choices: Yes ||| No + id: 4a98eef9-e316-44f0-b8e5-ecc7d7adcf54 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' reference: '' - 8da8d714-93f9-4f16-8115-270b98fd7c9a: !Template + 60028ba5-b56b-406b-8f78-5a8b38a1f885: !Template answer_choices: Yes ||| No - id: 8da8d714-93f9-4f16-8115-270b98fd7c9a + id: 60028ba5-b56b-406b-8f78-5a8b38a1f885 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -124,11 +70,11 @@ templates: metrics: - Accuracy original_task: false - name: Yes/No binary judgment for good sentences (choice order reversed) + name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' reference: '' - 9bb59a65-bf8a-4ec1-90d9-44956b266217: !Template + 658a5728-498d-4aac-8274-627a72a83d80: !Template answer_choices: A ||| B - id: 9bb59a65-bf8a-4ec1-90d9-44956b266217 + id: 658a5728-498d-4aac-8274-627a72a83d80 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -162,14 +108,14 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized + name: 'A/B choice randomized (choice order: A-B)' reference: '' - a68a23b5-fbc8-4fbf-9b55-ba6e7186e893: !Template + 6f9de6aa-a2fa-4d2d-8b18-6e620b20a19b: !Template answer_choices: null - id: a68a23b5-fbc8-4fbf-9b55-ba6e7186e893 + id: 6f9de6aa-a2fa-4d2d-8b18-6e620b20a19b jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -201,14 +147,14 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: random_choice_A_B reference: '' - bc28de55-b1c6-4efc-bbde-5b7893e145e5: !Template + 71a4569a-a82f-431b-b443-a5b953a3487c: !Template answer_choices: null - id: bc28de55-b1c6-4efc-bbde-5b7893e145e5 + id: 71a4569a-a82f-431b-b443-a5b953a3487c jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -240,14 +186,84 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: random_choice_1_2_2 + reference: '' + 9413e859-5ebe-43b9-a787-f2d2d14632ea: !Template + answer_choices: Yes ||| No + id: 9413e859-5ebe-43b9-a787-f2d2d14632ea + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' + reference: '' + a0905196-621f-47e5-82f7-15e7027accd4: !Template + answer_choices: Yes ||| No + id: a0905196-621f-47e5-82f7-15e7027accd4 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' + reference: '' + c9f91569-0add-4b92-b33c-b312f700c48d: !Template + answer_choices: A ||| B + id: c9f91569-0add-4b92-b33c-b312f700c48d + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice randomized (choice order: B-A)' reference: '' - c02161bc-a48d-4b12-8766-3f000b02ed1b: !Template + e885ab79-3348-427e-ab40-1714874db66c: !Template answer_choices: null - id: c02161bc-a48d-4b12-8766-3f000b02ed1b + id: e885ab79-3348-427e-ab40-1714874db66c jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -279,21 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B - reference: '' - f8c7071b-0938-48a3-aec0-7b33cb947bf1: !Template - answer_choices: Yes ||| No - id: f8c7071b-0938-48a3-aec0-7b33cb947bf1 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences + name: random_choice_A_B_2 reference: '' diff --git a/promptsource/templates/blimp/principle_A_reconstruction/templates.yaml b/promptsource/templates/blimp/principle_A_reconstruction/templates.yaml index 91f5d778f..8b6fba67e 100644 --- a/promptsource/templates/blimp/principle_A_reconstruction/templates.yaml +++ b/promptsource/templates/blimp/principle_A_reconstruction/templates.yaml @@ -1,9 +1,9 @@ dataset: blimp subset: principle_A_reconstruction templates: - 0db5847d-aff9-4f57-8acc-4648c6d3f7ba: !Template + 1f826642-2a9d-471f-adf6-90d6e4eef81a: !Template answer_choices: A ||| B - id: 0db5847d-aff9-4f57-8acc-4648c6d3f7ba + id: 1f826642-2a9d-471f-adf6-90d6e4eef81a jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -37,38 +37,37 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized + name: 'A/B choice randomized (choice order: A-B)' reference: '' - 34a7dafe-8bcb-49d8-9287-618ece2fd349: !Template - answer_choices: null - id: 34a7dafe-8bcb-49d8-9287-618ece2fd349 + 344f1444-76d0-477f-8fcf-3960fda4ac5a: !Template + answer_choices: A ||| B + id: 344f1444-76d0-477f-8fcf-3960fda4ac5a jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -76,11 +75,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: 'A/B choice randomized (choice order: B-A)' reference: '' - 34e77fcf-d6dd-47a2-8a9c-5e0802f8f30f: !Template + 364c32f7-567d-4eef-b632-42dd83f0203d: !Template answer_choices: Yes ||| No - id: 34e77fcf-d6dd-47a2-8a9c-5e0802f8f30f + id: 364c32f7-567d-4eef-b632-42dd83f0203d jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -92,37 +91,70 @@ templates: metrics: - Accuracy original_task: false - name: Yes/No binary judgment for good sentences + name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' reference: '' - 39205871-125d-40a7-8efa-4319539b9084: !Template - answer_choices: A ||| B - id: 39205871-125d-40a7-8efa-4319539b9084 + 43bba55d-06d6-4478-81ff-b20dee650280: !Template + answer_choices: Yes ||| No + id: 43bba55d-06d6-4478-81ff-b20dee650280 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' + reference: '' + 4e92194b-15f4-472e-b41d-e006b1d256f7: !Template + answer_choices: Yes ||| No + id: 4e92194b-15f4-472e-b41d-e006b1d256f7 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' + reference: '' + 5d38a794-6a00-467d-b582-589ddb0b9367: !Template + answer_choices: null + id: 5d38a794-6a00-467d-b582-589ddb0b9367 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| - {{ answer_choices[0] }} + {% if shuffled_order == 0 %} + + 1 {% else %} - {{ answer_choices[1] }} + 2 {% endif %}' metadata: !TemplateMetadata @@ -130,30 +162,14 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized (choice order reversed) - reference: '' - 5449f663-83f0-4031-9a00-51a08ef203a8: !Template - answer_choices: Yes ||| No - id: 5449f663-83f0-4031-9a00-51a08ef203a8 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences (choice order reversed) + name: random_choice_1_2 reference: '' - 91aa5feb-37e6-443f-90f1-b3ca15a79ea5: !Template + 60d55500-45c5-4d89-980d-68ba36e31819: !Template answer_choices: null - id: 91aa5feb-37e6-443f-90f1-b3ca15a79ea5 + id: 60d55500-45c5-4d89-980d-68ba36e31819 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -185,58 +201,26 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B - reference: '' - 941e5f65-136e-4dfb-be5f-708a90a14a71: !Template - answer_choices: Yes ||| No - id: 941e5f65-136e-4dfb-be5f-708a90a14a71 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences - reference: '' - 95a5c144-7ffa-4a81-9a63-17a1ea8c4e86: !Template - answer_choices: Yes ||| No - id: 95a5c144-7ffa-4a81-9a63-17a1ea8c4e86 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences (choice order reversed) + name: random_choice_A_B_2 reference: '' - aa72a26f-7e08-460b-96be-f2396dda07e2: !Template + bafaa33b-96ac-4bda-bffb-c3c5e113e3e9: !Template answer_choices: null - id: aa72a26f-7e08-460b-96be-f2396dda07e2 + id: bafaa33b-96ac-4bda-bffb-c3c5e113e3e9 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -244,11 +228,11 @@ templates: {% if shuffled_order == 0 %} - 1 + A {% else %} - 2 + B {% endif %}' metadata: !TemplateMetadata @@ -256,14 +240,14 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: random_choice_A_B reference: '' - cdb05cd3-5728-409d-aa66-1dc310be2db7: !Template + e1dc08d8-9353-4f8b-b0be-1a45c7a665d5: !Template answer_choices: null - id: cdb05cd3-5728-409d-aa66-1dc310be2db7 + id: e1dc08d8-9353-4f8b-b0be-1a45c7a665d5 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -295,5 +279,21 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: random_choice_1_2_2 + reference: '' + fdd1ca40-42db-4204-9a1f-f491c31cac93: !Template + answer_choices: Yes ||| No + id: fdd1ca40-42db-4204-9a1f-f491c31cac93 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' reference: '' diff --git a/promptsource/templates/blimp/regular_plural_subject_verb_agreement_1/templates.yaml b/promptsource/templates/blimp/regular_plural_subject_verb_agreement_1/templates.yaml index 856845f23..e4f843b79 100644 --- a/promptsource/templates/blimp/regular_plural_subject_verb_agreement_1/templates.yaml +++ b/promptsource/templates/blimp/regular_plural_subject_verb_agreement_1/templates.yaml @@ -1,35 +1,68 @@ dataset: blimp subset: regular_plural_subject_verb_agreement_1 templates: - 014a48b2-38c4-4078-8c56-1b8f6891c3c5: !Template - answer_choices: A ||| B - id: 014a48b2-38c4-4078-8c56-1b8f6891c3c5 + 036a4f23-0480-41e4-86a3-d97c633c6312: !Template + answer_choices: Yes ||| No + id: 036a4f23-0480-41e4-86a3-d97c633c6312 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' + reference: '' + 2190e8d7-6a9b-460b-99cd-a35f29ebd8cb: !Template + answer_choices: Yes ||| No + id: 2190e8d7-6a9b-460b-99cd-a35f29ebd8cb + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' + reference: '' + 235968d0-2297-4cf1-abf6-807b232c599e: !Template + answer_choices: null + id: 235968d0-2297-4cf1-abf6-807b232c599e jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| - {{ answer_choices[0] }} + {% if shuffled_order == 0 %} + + A {% else %} - {{ answer_choices[1] }} + B {% endif %}' metadata: !TemplateMetadata @@ -37,38 +70,69 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized + name: random_choice_A_B reference: '' - 2872aff2-8915-4372-b851-68c4080c2b75: !Template - answer_choices: null - id: 2872aff2-8915-4372-b851-68c4080c2b75 + 2fd30e23-066b-45d5-81cf-8735775dfa6e: !Template + answer_choices: Yes ||| No + id: 2fd30e23-066b-45d5-81cf-8735775dfa6e + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' + reference: '' + 32ab26ea-16ff-4bfb-bc88-46f9028cbc34: !Template + answer_choices: Yes ||| No + id: 32ab26ea-16ff-4bfb-bc88-46f9028cbc34 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' + reference: '' + 716ca2d5-8cbd-4271-aa66-c5e2f4de41e8: !Template + answer_choices: A ||| B + id: 716ca2d5-8cbd-4271-aa66-c5e2f4de41e8 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -76,11 +140,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: 'A/B choice randomized (choice order: B-A)' reference: '' - 29af5cb8-8c9f-4f12-a826-54efec4a6436: !Template + 87d503e6-f4e6-489e-8b4f-87d438be543b: !Template answer_choices: null - id: 29af5cb8-8c9f-4f12-a826-54efec4a6436 + id: 87d503e6-f4e6-489e-8b4f-87d438be543b jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -117,41 +181,9 @@ templates: original_task: true name: random_choice_1_2_2 reference: '' - 4f776d97-9192-4504-a702-b2ddbe0c0241: !Template - answer_choices: Yes ||| No - id: 4f776d97-9192-4504-a702-b2ddbe0c0241 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences (choice order reversed) - reference: '' - 662c389e-8f04-405d-a833-2e4a540de426: !Template - answer_choices: Yes ||| No - id: 662c389e-8f04-405d-a833-2e4a540de426 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences - reference: '' - 6ae71758-f2c6-436f-a33a-8c7169022c0a: !Template + 9e59aee4-eb66-4ab8-bf36-6c0d15e35cbd: !Template answer_choices: null - id: 6ae71758-f2c6-436f-a33a-8c7169022c0a + id: 9e59aee4-eb66-4ab8-bf36-6c0d15e35cbd jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer 1 or 2. @@ -188,41 +220,9 @@ templates: original_task: true name: random_choice_1_2 reference: '' - 91c0af57-fa62-427a-8835-ed4bff56e782: !Template - answer_choices: Yes ||| No - id: 91c0af57-fa62-427a-8835-ed4bff56e782 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences (choice order reversed) - reference: '' - d23cc85c-2a08-4323-ac81-475912fe4373: !Template - answer_choices: Yes ||| No - id: d23cc85c-2a08-4323-ac81-475912fe4373 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences - reference: '' - e7015947-5ae2-47d5-a2a6-9a9708323fd3: !Template + aad80196-c1f1-4244-b8dc-49bd17507fd7: !Template answer_choices: A ||| B - id: e7015947-5ae2-47d5-a2a6-9a9708323fd3 + id: aad80196-c1f1-4244-b8dc-49bd17507fd7 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -241,7 +241,7 @@ templates: {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order + Which one is a better sentence of English, A or B? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -256,14 +256,14 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized (choice order reversed) + name: 'A/B choice randomized (choice order: A-B)' reference: '' - f3f6cc90-6521-4154-bfc7-f86a417f91f7: !Template + fe59efd6-dfc0-4e6c-97fe-c5e6da764f5e: !Template answer_choices: null - id: f3f6cc90-6521-4154-bfc7-f86a417f91f7 + id: fe59efd6-dfc0-4e6c-97fe-c5e6da764f5e jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -295,5 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: random_choice_A_B_2 reference: '' diff --git a/promptsource/templates/blimp/regular_plural_subject_verb_agreement_2/templates.yaml b/promptsource/templates/blimp/regular_plural_subject_verb_agreement_2/templates.yaml index 3f83f9c11..a7df59118 100644 --- a/promptsource/templates/blimp/regular_plural_subject_verb_agreement_2/templates.yaml +++ b/promptsource/templates/blimp/regular_plural_subject_verb_agreement_2/templates.yaml @@ -1,9 +1,48 @@ dataset: blimp subset: regular_plural_subject_verb_agreement_2 templates: - 1b5c6c7d-ad79-4d6d-ae75-c19bff02b5a1: !Template + 14974aa7-8738-4dcd-9d8a-e3dbf879ab54: !Template + answer_choices: null + id: 14974aa7-8738-4dcd-9d8a-e3dbf879ab54 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2_2 + reference: '' + 24a9f478-f361-4cd0-a638-c500f887a582: !Template answer_choices: A ||| B - id: 1b5c6c7d-ad79-4d6d-ae75-c19bff02b5a1 + id: 24a9f478-f361-4cd0-a638-c500f887a582 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -37,11 +76,27 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized + name: 'A/B choice randomized (choice order: A-B)' reference: '' - 37b6791e-fcd6-4d2b-9b50-83ff3691e56f: !Template + 2ee2667d-4540-4ded-9276-ea3d9170034d: !Template answer_choices: Yes ||| No - id: 37b6791e-fcd6-4d2b-9b50-83ff3691e56f + id: 2ee2667d-4540-4ded-9276-ea3d9170034d + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' + reference: '' + 62d16043-8893-4a9b-8b2f-705015a2f7e8: !Template + answer_choices: Yes ||| No + id: 62d16043-8893-4a9b-8b2f-705015a2f7e8 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -53,11 +108,11 @@ templates: metrics: - Accuracy original_task: false - name: Yes/No binary judgment for good sentences + name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' reference: '' - 3af4390c-3587-47a9-a67c-f52fca276a89: !Template + 708f47c9-847d-4b98-8564-6c2584fe63bc: !Template answer_choices: Yes ||| No - id: 3af4390c-3587-47a9-a67c-f52fca276a89 + id: 708f47c9-847d-4b98-8564-6c2584fe63bc jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -69,77 +124,53 @@ templates: metrics: - Accuracy original_task: false - name: Yes/No binary judgment for bad sentences + name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' reference: '' - 4a05dfc5-2cc4-421b-9e10-64928c8d83c2: !Template - answer_choices: null - id: 4a05dfc5-2cc4-421b-9e10-64928c8d83c2 - jinja: '{% set shuffled_order = [0, 1] | random %} - - Between the two sentences, which one is grammatical? Please answer A or B. - - {% if shuffled_order == 0 %} - - A: {{ sentence_good }} - - B: {{ sentence_bad }} - - {% else %} - - A: {{ sentence_bad }} - - B: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - A - - {% else %} - - B + 73a4f13d-6010-4a08-80dc-c5cd9a30eba3: !Template + answer_choices: Yes ||| No + id: 73a4f13d-6010-4a08-80dc-c5cd9a30eba3 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - {% endif %}' + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: random_choice_A_B_2 + original_task: false + name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' reference: '' - 50d8598a-f818-41a8-ba21-550205105d2b: !Template - answer_choices: null - id: 50d8598a-f818-41a8-ba21-550205105d2b + a223dc66-85c0-4e9e-9cab-2d8d41897b09: !Template + answer_choices: A ||| B + id: a223dc66-85c0-4e9e-9cab-2d8d41897b09 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -147,30 +178,14 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: 'A/B choice randomized (choice order: B-A)' reference: '' - 64b4bc3c-af5e-4b54-90ec-34a81ecd249b: !Template - answer_choices: Yes ||| No - id: 64b4bc3c-af5e-4b54-90ec-34a81ecd249b - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences (choice order reversed) - reference: '' - 759b24f0-0f40-4123-ac11-f21b56a906df: !Template + bfa1b9c4-68a6-4f4d-8af3-b4b2cf4cee65: !Template answer_choices: null - id: 759b24f0-0f40-4123-ac11-f21b56a906df + id: bfa1b9c4-68a6-4f4d-8af3-b4b2cf4cee65 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -202,37 +217,38 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: random_choice_A_B_2 reference: '' - 9dfdce03-a939-414a-a465-617adc0f6460: !Template - answer_choices: A ||| B - id: 9dfdce03-a939-414a-a465-617adc0f6460 + c7f3752d-fc38-46f2-8f47-223108a3e042: !Template + answer_choices: null + id: c7f3752d-fc38-46f2-8f47-223108a3e042 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| - {{ answer_choices[0] }} + {% if shuffled_order == 0 %} + + 1 {% else %} - {{ answer_choices[1] }} + 2 {% endif %}' metadata: !TemplateMetadata @@ -240,42 +256,26 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized (choice order reversed) - reference: '' - d53410a3-0546-4af1-9df7-113a5a69925f: !Template - answer_choices: Yes ||| No - id: d53410a3-0546-4af1-9df7-113a5a69925f - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences (choice order reversed) + name: random_choice_1_2 reference: '' - dd7d0254-5f4f-4eee-92ff-4b860eb419d1: !Template + caa0cd60-20e8-44c5-aeda-516a8bf649f7: !Template answer_choices: null - id: dd7d0254-5f4f-4eee-92ff-4b860eb419d1 + id: caa0cd60-20e8-44c5-aeda-516a8bf649f7 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -283,11 +283,11 @@ templates: {% if shuffled_order == 0 %} - 1 + A {% else %} - 2 + B {% endif %}' metadata: !TemplateMetadata @@ -295,5 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: random_choice_A_B reference: '' diff --git a/promptsource/templates/blimp/sentential_negation_npi_licensor_present/templates.yaml b/promptsource/templates/blimp/sentential_negation_npi_licensor_present/templates.yaml index a055b776c..e86b8a7fe 100644 --- a/promptsource/templates/blimp/sentential_negation_npi_licensor_present/templates.yaml +++ b/promptsource/templates/blimp/sentential_negation_npi_licensor_present/templates.yaml @@ -1,9 +1,9 @@ dataset: blimp subset: sentential_negation_npi_licensor_present templates: - 1566b195-7965-4789-b33d-2d1d90355e49: !Template + 02730feb-390f-4f6e-91f5-742263d87ef3: !Template answer_choices: A ||| B - id: 1566b195-7965-4789-b33d-2d1d90355e49 + id: 02730feb-390f-4f6e-91f5-742263d87ef3 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -37,58 +37,42 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized + name: 'A/B choice randomized (choice order: A-B)' reference: '' - 1b7e5684-aeb4-4d8d-9d53-0cef4bb05eaf: !Template + 2af72d68-0fe9-492e-9532-1e0033764dd6: !Template answer_choices: Yes ||| No - id: 1b7e5684-aeb4-4d8d-9d53-0cef4bb05eaf + id: 2af72d68-0fe9-492e-9532-1e0033764dd6 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences - reference: '' - 1ff32cd6-cb3a-49a3-a87d-17813a312db1: !Template - answer_choices: Yes ||| No - id: 1ff32cd6-cb3a-49a3-a87d-17813a312db1 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: Yes/No binary judgment for good sentences (choice order reversed) + name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' reference: '' - 265a33b1-d0a8-488c-b0a5-871adbd49bbf: !Template + 5574f54b-db6e-4dd8-ba03-6c7d365688b2: !Template answer_choices: null - id: 265a33b1-d0a8-488c-b0a5-871adbd49bbf + id: 5574f54b-db6e-4dd8-ba03-6c7d365688b2 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -96,11 +80,11 @@ templates: {% if shuffled_order == 0 %} - A + 1 {% else %} - B + 2 {% endif %}' metadata: !TemplateMetadata @@ -108,14 +92,30 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: random_choice_1_2 + reference: '' + 6357d10b-82a3-4f91-a79f-a905542c75d4: !Template + answer_choices: Yes ||| No + id: 6357d10b-82a3-4f91-a79f-a905542c75d4 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' reference: '' - 379fc292-eb52-442a-b900-8b8466d50199: !Template + 6d18ce1d-ca55-44fd-9041-1f27bcef0609: !Template answer_choices: null - id: 379fc292-eb52-442a-b900-8b8466d50199 + id: 6d18ce1d-ca55-44fd-9041-1f27bcef0609 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -147,27 +147,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 - reference: '' - 6913d325-20d1-4712-8a9e-556ede3ddb7b: !Template - answer_choices: Yes ||| No - id: 6913d325-20d1-4712-8a9e-556ede3ddb7b - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences + name: random_choice_1_2_2 reference: '' - 8ed2111e-04fe-4d9d-82e0-9a0f0d10c14f: !Template + 6f84156e-22ae-4c07-8e3e-4006e39fcf99: !Template answer_choices: A ||| B - id: 8ed2111e-04fe-4d9d-82e0-9a0f0d10c14f + id: 6f84156e-22ae-4c07-8e3e-4006e39fcf99 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -201,30 +185,14 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized (choice order reversed) - reference: '' - 9f30c4ec-ee1c-4b39-bc13-8aba03d1f937: !Template - answer_choices: Yes ||| No - id: 9f30c4ec-ee1c-4b39-bc13-8aba03d1f937 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences (choice order reversed) + name: 'A/B choice randomized (choice order: B-A)' reference: '' - b2dbbf76-a5b5-4610-b386-86656690cf3f: !Template + 721da667-24b0-4f42-be97-3974b69a0ff1: !Template answer_choices: null - id: b2dbbf76-a5b5-4610-b386-86656690cf3f + id: 721da667-24b0-4f42-be97-3974b69a0ff1 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -256,26 +224,42 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: random_choice_A_B reference: '' - dab171c5-f782-4936-b33c-aefc9b34c6a3: !Template + bf0fe469-8193-412e-a789-64028bde8aaf: !Template + answer_choices: Yes ||| No + id: bf0fe469-8193-412e-a789-64028bde8aaf + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' + reference: '' + cfe57c2a-1359-485d-870d-7eafa36e86cb: !Template answer_choices: null - id: dab171c5-f782-4936-b33c-aefc9b34c6a3 + id: cfe57c2a-1359-485d-870d-7eafa36e86cb jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -283,11 +267,11 @@ templates: {% if shuffled_order == 0 %} - 1 + A {% else %} - 2 + B {% endif %}' metadata: !TemplateMetadata @@ -295,5 +279,21 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: random_choice_A_B_2 + reference: '' + e6110306-ae02-43ff-b136-42b1dcc3a3d1: !Template + answer_choices: Yes ||| No + id: e6110306-ae02-43ff-b136-42b1dcc3a3d1 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' reference: '' diff --git a/promptsource/templates/blimp/sentential_negation_npi_scope/templates.yaml b/promptsource/templates/blimp/sentential_negation_npi_scope/templates.yaml index 8b55678ef..6607c07a2 100644 --- a/promptsource/templates/blimp/sentential_negation_npi_scope/templates.yaml +++ b/promptsource/templates/blimp/sentential_negation_npi_scope/templates.yaml @@ -1,51 +1,36 @@ dataset: blimp subset: sentential_negation_npi_scope templates: - 0f7b0f57-f067-4bc2-b028-408443f36ffa: !Template - answer_choices: Yes ||| No - id: 0f7b0f57-f067-4bc2-b028-408443f36ffa - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences (choice order reversed) - reference: '' - 2c466776-03e8-4029-bb49-9b96f4f3032d: !Template - answer_choices: A ||| B - id: 2c466776-03e8-4029-bb49-9b96f4f3032d + 0724eef2-aff1-4c53-bd9a-8c76713f4936: !Template + answer_choices: null + id: 0724eef2-aff1-4c53-bd9a-8c76713f4936 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| - {{ answer_choices[0] }} + {% if shuffled_order == 0 %} + + 1 {% else %} - {{ answer_choices[1] }} + 2 {% endif %}' metadata: !TemplateMetadata @@ -53,31 +38,15 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized (choice order reversed) - reference: '' - 39e28834-c2e7-4cdb-b0cd-360b64aaae5f: !Template - answer_choices: Yes ||| No - id: 39e28834-c2e7-4cdb-b0cd-360b64aaae5f - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences + name: random_choice_1_2_2 reference: '' - 44d0d846-2859-450a-b546-2872a20013e3: !Template + 2dc5b5e4-60f4-4f6e-97ca-715d00f3326e: !Template answer_choices: Yes ||| No - id: 44d0d846-2859-450a-b546-2872a20013e3 + id: 2dc5b5e4-60f4-4f6e-97ca-715d00f3326e jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata @@ -85,27 +54,11 @@ templates: metrics: - Accuracy original_task: false - name: Yes/No binary judgment for bad sentences - reference: '' - a88e1e26-b4e4-4f6d-8582-ad085c44bf54: !Template - answer_choices: Yes ||| No - id: a88e1e26-b4e4-4f6d-8582-ad085c44bf54 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences (choice order reversed) + name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' reference: '' - aa761950-8c2b-47c9-9203-c0bc15c64d14: !Template + 2df0afd1-8c9a-4b0a-895a-3b7b93b22205: !Template answer_choices: A ||| B - id: aa761950-8c2b-47c9-9203-c0bc15c64d14 + id: 2df0afd1-8c9a-4b0a-895a-3b7b93b22205 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -139,26 +92,26 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized + name: 'A/B choice randomized (choice order: A-B)' reference: '' - b7dcb54b-d02a-4c85-9e52-256ba5daaf8c: !Template + 6817fe38-c385-4902-91e1-bb424e2a6ee4: !Template answer_choices: null - id: b7dcb54b-d02a-4c85-9e52-256ba5daaf8c + id: 6817fe38-c385-4902-91e1-bb424e2a6ee4 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -166,11 +119,11 @@ templates: {% if shuffled_order == 0 %} - A + 1 {% else %} - B + 2 {% endif %}' metadata: !TemplateMetadata @@ -178,14 +131,30 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: random_choice_1_2 + reference: '' + 8c1cfffd-03e9-40d4-b86b-9ac8b51c0a0c: !Template + answer_choices: Yes ||| No + id: 8c1cfffd-03e9-40d4-b86b-9ac8b51c0a0c + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' reference: '' - b8087f0e-30ca-4a7b-ad59-0f143eb3ec15: !Template + 908db614-72a9-488b-a880-c6656ddee630: !Template answer_choices: null - id: b8087f0e-30ca-4a7b-ad59-0f143eb3ec15 + id: 908db614-72a9-488b-a880-c6656ddee630 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -217,38 +186,53 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: random_choice_A_B_2 reference: '' - eac257fa-d2f6-4b36-9805-6aa3e68c4ab1: !Template - answer_choices: null - id: eac257fa-d2f6-4b36-9805-6aa3e68c4ab1 + a6484d4c-158f-4069-8ced-af9a48f49e15: !Template + answer_choices: Yes ||| No + id: a6484d4c-158f-4069-8ced-af9a48f49e15 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' + reference: '' + b2b3d98c-78fb-41b2-ab3a-2228f3743df7: !Template + answer_choices: A ||| B + id: b2b3d98c-78fb-41b2-ab3a-2228f3743df7 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -256,26 +240,42 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: 'A/B choice randomized (choice order: B-A)' + reference: '' + c30bc093-6b91-4da4-97ba-5369cac2d159: !Template + answer_choices: Yes ||| No + id: c30bc093-6b91-4da4-97ba-5369cac2d159 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' reference: '' - f4785d78-7953-4332-adc9-89d6df976106: !Template + e2f937fb-bfa6-4c76-ada2-fb7b0f12a28b: !Template answer_choices: null - id: f4785d78-7953-4332-adc9-89d6df976106 + id: e2f937fb-bfa6-4c76-ada2-fb7b0f12a28b jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -283,11 +283,11 @@ templates: {% if shuffled_order == 0 %} - 1 + A {% else %} - 2 + B {% endif %}' metadata: !TemplateMetadata @@ -295,5 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: random_choice_A_B reference: '' diff --git a/promptsource/templates/blimp/sentential_subject_island/templates.yaml b/promptsource/templates/blimp/sentential_subject_island/templates.yaml index cd8cd07c7..df4e65016 100644 --- a/promptsource/templates/blimp/sentential_subject_island/templates.yaml +++ b/promptsource/templates/blimp/sentential_subject_island/templates.yaml @@ -1,25 +1,48 @@ dataset: blimp subset: sentential_subject_island templates: - 16fdd0d2-8afa-4df4-a06c-c97c8e9f766d: !Template - answer_choices: Yes ||| No - id: 16fdd0d2-8afa-4df4-a06c-c97c8e9f766d - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + 008f1f04-9c60-4100-9596-1ae92b5f369d: !Template + answer_choices: null + id: 008f1f04-9c60-4100-9596-1ae92b5f369d + jinja: '{% set shuffled_order = [0, 1] | random %} - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences (choice order reversed) + original_task: true + name: random_choice_A_B_2 reference: '' - 1f0def75-b22b-4ced-bb79-86a3095cfb7a: !Template + 14b7cba4-dbee-4fd1-a365-417cf47567ba: !Template answer_choices: null - id: 1f0def75-b22b-4ced-bb79-86a3095cfb7a + id: 14b7cba4-dbee-4fd1-a365-417cf47567ba jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer 1 or 2. @@ -56,25 +79,9 @@ templates: original_task: true name: random_choice_1_2 reference: '' - 29a052e7-6f9a-4ddd-902b-ba2aa7d779b6: !Template - answer_choices: Yes ||| No - id: 29a052e7-6f9a-4ddd-902b-ba2aa7d779b6 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences (choice order reversed) - reference: '' - 34f43d45-61df-43aa-ae8d-43a58ea464fe: !Template + 301ca326-ff37-4415-8da8-8eff36300b83: !Template answer_choices: Yes ||| No - id: 34f43d45-61df-43aa-ae8d-43a58ea464fe + id: 301ca326-ff37-4415-8da8-8eff36300b83 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -86,14 +93,14 @@ templates: metrics: - Accuracy original_task: false - name: Yes/No binary judgment for bad sentences + name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' reference: '' - 36bddd3c-678e-4fad-9d1d-e27ff2d072a5: !Template + 3e1817a6-288f-46fd-9304-a52dc801fcce: !Template answer_choices: null - id: 36bddd3c-678e-4fad-9d1d-e27ff2d072a5 + id: 3e1817a6-288f-46fd-9304-a52dc801fcce jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -125,64 +132,42 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: random_choice_A_B reference: '' - 70a1ae91-8cc3-4465-8b91-6b7a26b2b88a: !Template - answer_choices: A ||| B - id: 70a1ae91-8cc3-4465-8b91-6b7a26b2b88a - jinja: '{% set shuffled_order = [0, 1] | random %} - - Read the following two sentences. - - {% if shuffled_order == 0 %} - - Sentence A: {{ sentence_good }} - - Sentence B: {{ sentence_bad }} - - {% else %} - - Sentence A: {{ sentence_bad }} - - Sentence B: {{ sentence_good }} - - {% endif %} - - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} + 3f717291-e579-4ffa-805e-185856f954fb: !Template + answer_choices: Yes ||| No + id: 3f717291-e579-4ffa-805e-185856f954fb + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - {% endif %}' + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: A/B choice randomized (choice order reversed) + original_task: false + name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' reference: '' - 7a98a195-9982-4cc8-9958-6a673b218893: !Template + 4cddcaf0-86dc-4dfe-b3ab-9edd07b6fd23: !Template answer_choices: null - id: 7a98a195-9982-4cc8-9958-6a673b218893 + id: 4cddcaf0-86dc-4dfe-b3ab-9edd07b6fd23 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -190,11 +175,11 @@ templates: {% if shuffled_order == 0 %} - A + 1 {% else %} - B + 2 {% endif %}' metadata: !TemplateMetadata @@ -202,11 +187,27 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: random_choice_1_2_2 reference: '' - 7cd25f9a-0a2d-452b-a7b3-243d232e5ddd: !Template + 5728be55-6b7a-46c6-a2e9-fe0d2ff462e1: !Template + answer_choices: Yes ||| No + id: 5728be55-6b7a-46c6-a2e9-fe0d2ff462e1 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' + reference: '' + 8516d3eb-dc0d-4eb3-8324-544a7ca7e69a: !Template answer_choices: A ||| B - id: 7cd25f9a-0a2d-452b-a7b3-243d232e5ddd + id: 8516d3eb-dc0d-4eb3-8324-544a7ca7e69a jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -225,7 +226,7 @@ templates: {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order + Which one is a better sentence of English, B or A? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -240,38 +241,37 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized + name: 'A/B choice randomized (choice order: B-A)' reference: '' - 904692c6-2d60-4721-9eb2-c36d91794716: !Template - answer_choices: null - id: 904692c6-2d60-4721-9eb2-c36d91794716 + 9239fcdb-429a-4c1f-9863-c7f42d3124a0: !Template + answer_choices: A ||| B + id: 9239fcdb-429a-4c1f-9863-c7f42d3124a0 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -279,15 +279,15 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: 'A/B choice randomized (choice order: A-B)' reference: '' - fed02fad-6382-4f21-a66a-98ec3a9ef084: !Template + cdb3bc9b-2645-415b-b5a5-6e607f6bae8d: !Template answer_choices: Yes ||| No - id: fed02fad-6382-4f21-a66a-98ec3a9ef084 + id: cdb3bc9b-2645-415b-b5a5-6e607f6bae8d jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -295,5 +295,5 @@ templates: metrics: - Accuracy original_task: false - name: Yes/No binary judgment for good sentences + name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' reference: '' diff --git a/promptsource/templates/blimp/superlative_quantifiers_1/templates.yaml b/promptsource/templates/blimp/superlative_quantifiers_1/templates.yaml index cae680695..155bd5401 100644 --- a/promptsource/templates/blimp/superlative_quantifiers_1/templates.yaml +++ b/promptsource/templates/blimp/superlative_quantifiers_1/templates.yaml @@ -1,9 +1,64 @@ dataset: blimp subset: superlative_quantifiers_1 templates: - 06382d63-4b72-483b-ae6c-9861bf24f81c: !Template + 4f251422-3bf3-448f-90c6-b3d7ff0d8cea: !Template + answer_choices: null + id: 4f251422-3bf3-448f-90c6-b3d7ff0d8cea + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_A_B + reference: '' + 5fc50de0-8349-4007-b7d4-a9cae68ad67f: !Template + answer_choices: Yes ||| No + id: 5fc50de0-8349-4007-b7d4-a9cae68ad67f + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' + reference: '' + 69c7fc58-4bf3-4ab4-b691-3ee9138e3e3a: !Template answer_choices: Yes ||| No - id: 06382d63-4b72-483b-ae6c-9861bf24f81c + id: 69c7fc58-4bf3-4ab4-b691-3ee9138e3e3a jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -15,14 +70,14 @@ templates: metrics: - Accuracy original_task: false - name: Yes/No binary judgment for bad sentences + name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' reference: '' - 159b2baa-694b-401b-820f-bba21fce82cf: !Template + 872c75f1-7513-4a56-8d99-351a5c62a80d: !Template answer_choices: null - id: 159b2baa-694b-401b-820f-bba21fce82cf + id: 872c75f1-7513-4a56-8d99-351a5c62a80d jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -54,11 +109,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: random_choice_A_B_2 reference: '' - 1f275906-f740-4f5c-a136-2f7beaa19975: !Template + ae0069dc-d181-4e60-8c77-629d8b8c3c24: !Template answer_choices: A ||| B - id: 1f275906-f740-4f5c-a136-2f7beaa19975 + id: ae0069dc-d181-4e60-8c77-629d8b8c3c24 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -77,7 +132,7 @@ templates: {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order + Which one is a better sentence of English, A or B? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -92,43 +147,27 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized (choice order reversed) - reference: '' - 2b7e9a30-fb60-4980-beb2-6ec591dc470d: !Template - answer_choices: Yes ||| No - id: 2b7e9a30-fb60-4980-beb2-6ec591dc470d - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences + name: 'A/B choice randomized (choice order: A-B)' reference: '' - 60574278-67e2-43b7-802a-b77ada04d832: !Template + b563c6ed-392b-402f-a8ae-ea2ca019446e: !Template answer_choices: Yes ||| No - id: 60574278-67e2-43b7-802a-b77ada04d832 + id: b563c6ed-392b-402f-a8ae-ea2ca019446e jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: Yes/No binary judgment for good sentences (choice order reversed) + name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' reference: '' - 62cc186f-85fb-410a-a980-88d6aca187da: !Template + dab9ed24-e1dd-452e-8999-5231909b5f36: !Template answer_choices: null - id: 62cc186f-85fb-410a-a980-88d6aca187da + id: dab9ed24-e1dd-452e-8999-5231909b5f36 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer 1 or 2. @@ -165,9 +204,9 @@ templates: original_task: true name: random_choice_1_2 reference: '' - 6ca59cd4-3888-423e-bc5d-ab8380ed7850: !Template + f474ffc2-075c-4ff0-af37-491a8034cec5: !Template answer_choices: A ||| B - id: 6ca59cd4-3888-423e-bc5d-ab8380ed7850 + id: f474ffc2-075c-4ff0-af37-491a8034cec5 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -186,7 +225,7 @@ templates: {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order + Which one is a better sentence of English, B or A? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -201,81 +240,42 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized - reference: '' - 876a0627-e0e9-4b42-8dec-662e97f14850: !Template - answer_choices: null - id: 876a0627-e0e9-4b42-8dec-662e97f14850 - jinja: '{% set shuffled_order = [0, 1] | random %} - - Between the two sentences, which one is grammatical? Please answer 1 or 2. - - {% if shuffled_order == 0 %} - - 1: {{ sentence_good }} - - 2: {{ sentence_bad }} - - {% else %} - - 1: {{ sentence_bad }} - - 2: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - 1 - - {% else %} - - 2 - - {% endif %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: random_choice_1_2_2 + name: 'A/B choice randomized (choice order: B-A)' reference: '' - b248fd0b-a990-4db7-a26c-2325729bbbee: !Template + f8847769-33b1-42cd-ae19-b1a537bfa066: !Template answer_choices: Yes ||| No - id: b248fd0b-a990-4db7-a26c-2325729bbbee + id: f8847769-33b1-42cd-ae19-b1a537bfa066 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: Yes/No binary judgment for bad sentences (choice order reversed) + name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' reference: '' - ee010f7d-c4e2-4640-8d00-01e2517a78be: !Template + fd169ab7-3b97-415c-a485-a5a71f915bf4: !Template answer_choices: null - id: ee010f7d-c4e2-4640-8d00-01e2517a78be + id: fd169ab7-3b97-415c-a485-a5a71f915bf4 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -283,11 +283,11 @@ templates: {% if shuffled_order == 0 %} - A + 1 {% else %} - B + 2 {% endif %}' metadata: !TemplateMetadata @@ -295,5 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: random_choice_1_2_2 reference: '' diff --git a/promptsource/templates/blimp/superlative_quantifiers_2/templates.yaml b/promptsource/templates/blimp/superlative_quantifiers_2/templates.yaml index c5c9982c2..f886173d3 100644 --- a/promptsource/templates/blimp/superlative_quantifiers_2/templates.yaml +++ b/promptsource/templates/blimp/superlative_quantifiers_2/templates.yaml @@ -1,36 +1,35 @@ dataset: blimp subset: superlative_quantifiers_2 templates: - 0e6b233c-6dc1-4e40-90e2-45c124dba2a6: !Template - answer_choices: null - id: 0e6b233c-6dc1-4e40-90e2-45c124dba2a6 + 1149a7af-e509-411b-80be-192905e047d4: !Template + answer_choices: A ||| B + id: 1149a7af-e509-411b-80be-192905e047d4 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -38,26 +37,26 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: 'A/B choice randomized (choice order: A-B)' reference: '' - 216e6b95-edf9-432e-a312-a56ba3a7e338: !Template + 263d68e2-e44e-4f8f-8b1d-487b2aa70b3a: !Template answer_choices: null - id: 216e6b95-edf9-432e-a312-a56ba3a7e338 + id: 263d68e2-e44e-4f8f-8b1d-487b2aa70b3a jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -65,11 +64,11 @@ templates: {% if shuffled_order == 0 %} - 1 + A {% else %} - 2 + B {% endif %}' metadata: !TemplateMetadata @@ -77,11 +76,27 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: random_choice_A_B + reference: '' + 32119cbc-b09a-4b0d-b9c4-687b55e7c696: !Template + answer_choices: Yes ||| No + id: 32119cbc-b09a-4b0d-b9c4-687b55e7c696 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' reference: '' - 26285bf8-75dd-46b5-94ba-1740a1e43143: !Template + 522e13a1-fcd9-4540-bba0-a4439715cc1d: !Template answer_choices: null - id: 26285bf8-75dd-46b5-94ba-1740a1e43143 + id: 522e13a1-fcd9-4540-bba0-a4439715cc1d jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -118,25 +133,9 @@ templates: original_task: true name: random_choice_1_2_2 reference: '' - 3a280cc5-2ffd-45d3-a64d-a8424540d9fb: !Template - answer_choices: Yes ||| No - id: 3a280cc5-2ffd-45d3-a64d-a8424540d9fb - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences (choice order reversed) - reference: '' - 541fa01a-73f6-4ddc-8861-50d4fcc949ab: !Template + 59110d8e-f7f9-4d26-b72d-3fbfc9ed9970: !Template answer_choices: A ||| B - id: 541fa01a-73f6-4ddc-8861-50d4fcc949ab + id: 59110d8e-f7f9-4d26-b72d-3fbfc9ed9970 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -170,37 +169,38 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized (choice order reversed) + name: 'A/B choice randomized (choice order: B-A)' reference: '' - 6c7883a7-519f-4b7b-8646-b8c37f0247d1: !Template - answer_choices: A ||| B - id: 6c7883a7-519f-4b7b-8646-b8c37f0247d1 + 903bf9f1-4b0b-4f05-acb6-fb2bcdb876c0: !Template + answer_choices: null + id: 903bf9f1-4b0b-4f05-acb6-fb2bcdb876c0 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| - {{ answer_choices[0] }} + {% if shuffled_order == 0 %} + + A {% else %} - {{ answer_choices[1] }} + B {% endif %}' metadata: !TemplateMetadata @@ -208,15 +208,15 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized + name: random_choice_A_B_2 reference: '' - 70808c31-fc16-4343-a107-bed61889e83f: !Template + ad315691-8475-426d-b61c-754b4cfe0b4d: !Template answer_choices: Yes ||| No - id: 70808c31-fc16-4343-a107-bed61889e83f + id: ad315691-8475-426d-b61c-754b4cfe0b4d jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata @@ -224,42 +224,26 @@ templates: metrics: - Accuracy original_task: false - name: Yes/No binary judgment for bad sentences + name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' reference: '' - a913f701-6df7-4dcc-9000-735a07e65ef5: !Template - answer_choices: Yes ||| No - id: a913f701-6df7-4dcc-9000-735a07e65ef5 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences - reference: '' - ade2bfa0-df0e-4e13-948c-57e11f33e6b3: !Template + c0d98b6b-c484-4537-81be-292d6ef0496b: !Template answer_choices: null - id: ade2bfa0-df0e-4e13-948c-57e11f33e6b3 + id: c0d98b6b-c484-4537-81be-292d6ef0496b jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -267,11 +251,11 @@ templates: {% if shuffled_order == 0 %} - A + 1 {% else %} - B + 2 {% endif %}' metadata: !TemplateMetadata @@ -279,11 +263,27 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: random_choice_1_2 + reference: '' + d74fba72-6ac3-455f-9459-ebf52c93e5a8: !Template + answer_choices: Yes ||| No + id: d74fba72-6ac3-455f-9459-ebf52c93e5a8 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' reference: '' - e84c0767-2ec6-4067-9401-b9335e013d39: !Template + f9d185b6-78ed-4b5c-97cc-f73c2242e3a2: !Template answer_choices: Yes ||| No - id: e84c0767-2ec6-4067-9401-b9335e013d39 + id: f9d185b6-78ed-4b5c-97cc-f73c2242e3a2 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -295,5 +295,5 @@ templates: metrics: - Accuracy original_task: false - name: Yes/No binary judgment for good sentences (choice order reversed) + name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' reference: '' diff --git a/promptsource/templates/blimp/tough_vs_raising_1/templates.yaml b/promptsource/templates/blimp/tough_vs_raising_1/templates.yaml index 59fc67c85..4439add8e 100644 --- a/promptsource/templates/blimp/tough_vs_raising_1/templates.yaml +++ b/promptsource/templates/blimp/tough_vs_raising_1/templates.yaml @@ -1,35 +1,36 @@ dataset: blimp subset: tough_vs_raising_1 templates: - 1686d3fb-896a-4a5e-9899-9f42d3c7fbc7: !Template - answer_choices: A ||| B - id: 1686d3fb-896a-4a5e-9899-9f42d3c7fbc7 + 0528d81f-ab0b-47c8-a7bd-80dc4b6d84d8: !Template + answer_choices: null + id: 0528d81f-ab0b-47c8-a7bd-80dc4b6d84d8 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| - {{ answer_choices[0] }} + {% if shuffled_order == 0 %} + + 1 {% else %} - {{ answer_choices[1] }} + 2 {% endif %}' metadata: !TemplateMetadata @@ -37,42 +38,26 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized (choice order reversed) - reference: '' - 4abbdd3c-141c-4457-a652-e8f854e1f6b3: !Template - answer_choices: Yes ||| No - id: 4abbdd3c-141c-4457-a652-e8f854e1f6b3 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences (choice order reversed) + name: random_choice_1_2_2 reference: '' - 51185250-89ac-415c-8c2c-aac5322257b1: !Template + 0745a26d-37a7-4ef1-b544-26994e3eb1d2: !Template answer_choices: null - id: 51185250-89ac-415c-8c2c-aac5322257b1 + id: 0745a26d-37a7-4ef1-b544-26994e3eb1d2 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -80,11 +65,11 @@ templates: {% if shuffled_order == 0 %} - 1 + A {% else %} - 2 + B {% endif %}' metadata: !TemplateMetadata @@ -92,11 +77,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: random_choice_A_B_2 reference: '' - 79bb6de3-5378-4a3d-9d8d-d5e24f63f8a0: !Template + 183a789a-ec55-40e1-b0af-2f30532ec07b: !Template answer_choices: null - id: 79bb6de3-5378-4a3d-9d8d-d5e24f63f8a0 + id: 183a789a-ec55-40e1-b0af-2f30532ec07b jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer 1 or 2. @@ -133,36 +118,35 @@ templates: original_task: true name: random_choice_1_2 reference: '' - 80216836-1950-4484-93a5-09e74b99ec5d: !Template - answer_choices: null - id: 80216836-1950-4484-93a5-09e74b99ec5d + 1e641e47-4f12-4351-9f93-467e6ce44a02: !Template + answer_choices: A ||| B + id: 1e641e47-4f12-4351-9f93-467e6ce44a02 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -170,27 +154,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 - reference: '' - 81139d64-3df6-4a97-b823-d3d22e41cc21: !Template - answer_choices: Yes ||| No - id: 81139d64-3df6-4a97-b823-d3d22e41cc21 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences (choice order reversed) + name: 'A/B choice randomized (choice order: A-B)' reference: '' - 95f298f3-b243-4b5c-bb12-070b21165d02: !Template + 7b5b1b06-7029-4e51-a501-385d8e5a6f8a: !Template answer_choices: null - id: 95f298f3-b243-4b5c-bb12-070b21165d02 + id: 7b5b1b06-7029-4e51-a501-385d8e5a6f8a jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer A or B. @@ -227,9 +195,57 @@ templates: original_task: true name: random_choice_A_B reference: '' - b367ca33-aabe-43d0-be2e-089f805f4219: !Template + 8cfb32b8-4459-4b89-8285-780d457b9f72: !Template + answer_choices: Yes ||| No + id: 8cfb32b8-4459-4b89-8285-780d457b9f72 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' + reference: '' + 9306abb3-5707-4fb8-8ad7-4fbd5329906d: !Template + answer_choices: Yes ||| No + id: 9306abb3-5707-4fb8-8ad7-4fbd5329906d + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' + reference: '' + 9b13b45f-1b3c-4a40-ae7f-b1ab8b57a781: !Template + answer_choices: Yes ||| No + id: 9b13b45f-1b3c-4a40-ae7f-b1ab8b57a781 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' + reference: '' + af7dd32a-7696-447d-addb-3fad35c9284f: !Template answer_choices: A ||| B - id: b367ca33-aabe-43d0-be2e-089f805f4219 + id: af7dd32a-7696-447d-addb-3fad35c9284f jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -248,7 +264,7 @@ templates: {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order + Which one is a better sentence of English, B or A? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -263,27 +279,11 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized - reference: '' - b73e4e71-2b5a-4bae-98a5-f71ac94c44ef: !Template - answer_choices: Yes ||| No - id: b73e4e71-2b5a-4bae-98a5-f71ac94c44ef - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences + name: 'A/B choice randomized (choice order: B-A)' reference: '' - e7bf6d42-f4be-4b9d-bee7-cfd05b09a561: !Template + e64a405e-504b-4de6-98c7-567c04986b6b: !Template answer_choices: Yes ||| No - id: e7bf6d42-f4be-4b9d-bee7-cfd05b09a561 + id: e64a405e-504b-4de6-98c7-567c04986b6b jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -295,5 +295,5 @@ templates: metrics: - Accuracy original_task: false - name: Yes/No binary judgment for bad sentences + name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' reference: '' diff --git a/promptsource/templates/blimp/tough_vs_raising_2/templates.yaml b/promptsource/templates/blimp/tough_vs_raising_2/templates.yaml index 98acfe535..dd88093dd 100644 --- a/promptsource/templates/blimp/tough_vs_raising_2/templates.yaml +++ b/promptsource/templates/blimp/tough_vs_raising_2/templates.yaml @@ -1,25 +1,9 @@ dataset: blimp subset: tough_vs_raising_2 templates: - 14e649e0-966e-4037-ae98-a619018e0104: !Template - answer_choices: Yes ||| No - id: 14e649e0-966e-4037-ae98-a619018e0104 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences (choice order reversed) - reference: '' - 2a18d5a6-e07e-482e-9e66-b05a6ab05040: !Template + 0d54febd-41ae-4fbd-8ca7-ef5aac51453c: !Template answer_choices: A ||| B - id: 2a18d5a6-e07e-482e-9e66-b05a6ab05040 + id: 0d54febd-41ae-4fbd-8ca7-ef5aac51453c jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -38,7 +22,7 @@ templates: {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order + Which one is a better sentence of English, B or A? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -53,26 +37,26 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized + name: 'A/B choice randomized (choice order: B-A)' reference: '' - 641ac8f7-8edd-4a87-abd5-8fdff0da9d19: !Template + 22580638-cf0c-46c2-97dd-7ffea779d0a6: !Template answer_choices: null - id: 641ac8f7-8edd-4a87-abd5-8fdff0da9d19 + id: 22580638-cf0c-46c2-97dd-7ffea779d0a6 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -80,11 +64,11 @@ templates: {% if shuffled_order == 0 %} - A + 1 {% else %} - B + 2 {% endif %}' metadata: !TemplateMetadata @@ -92,15 +76,15 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: random_choice_1_2_2 reference: '' - 7a632207-6987-411b-a5d4-57874a2caa82: !Template + 581810cc-0477-4179-a8b0-65954d68e802: !Template answer_choices: Yes ||| No - id: 7a632207-6987-411b-a5d4-57874a2caa82 + id: 581810cc-0477-4179-a8b0-65954d68e802 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata @@ -108,27 +92,50 @@ templates: metrics: - Accuracy original_task: false - name: Yes/No binary judgment for bad sentences + name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' reference: '' - 8031c00b-e407-4bba-9944-f2d1ed5d0839: !Template - answer_choices: Yes ||| No - id: 8031c00b-e407-4bba-9944-f2d1ed5d0839 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + 5a1dfbf7-feb3-4457-be61-3c095b91e426: !Template + answer_choices: null + id: 5a1dfbf7-feb3-4457-be61-3c095b91e426 + jinja: '{% set shuffled_order = [0, 1] | random %} - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences (choice order reversed) + original_task: true + name: random_choice_1_2 reference: '' - 935bcac4-f021-4532-919b-302db1f4282c: !Template + 5a9680a8-eaaf-48ef-909a-0260461629fa: !Template answer_choices: null - id: 935bcac4-f021-4532-919b-302db1f4282c + id: 5a9680a8-eaaf-48ef-909a-0260461629fa jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer A or B. @@ -165,9 +172,9 @@ templates: original_task: true name: random_choice_A_B reference: '' - 99d65ae4-c014-41eb-a3f1-a2b608572668: !Template + 7ae170fa-ddac-4d62-bd23-5af95cbd8b59: !Template answer_choices: A ||| B - id: 99d65ae4-c014-41eb-a3f1-a2b608572668 + id: 7ae170fa-ddac-4d62-bd23-5af95cbd8b59 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -186,7 +193,7 @@ templates: {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order + Which one is a better sentence of English, A or B? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -201,42 +208,26 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized (choice order reversed) - reference: '' - 9fc0d7e6-7b4f-4614-84fc-5cd0c6882513: !Template - answer_choices: Yes ||| No - id: 9fc0d7e6-7b4f-4614-84fc-5cd0c6882513 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences + name: 'A/B choice randomized (choice order: A-B)' reference: '' - ad13698d-af51-4145-b12f-e6809075792a: !Template + 9845e5d4-cb4e-4149-aabe-e4a73ad5b93a: !Template answer_choices: null - id: ad13698d-af51-4145-b12f-e6809075792a + id: 9845e5d4-cb4e-4149-aabe-e4a73ad5b93a jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -244,11 +235,11 @@ templates: {% if shuffled_order == 0 %} - 1 + A {% else %} - 2 + B {% endif %}' metadata: !TemplateMetadata @@ -256,44 +247,53 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: random_choice_A_B_2 reference: '' - f081fe14-16f8-40a3-95fe-cf5b83c81afd: !Template - answer_choices: null - id: f081fe14-16f8-40a3-95fe-cf5b83c81afd - jinja: '{% set shuffled_order = [0, 1] | random %} - - Between the two sentences, which one is grammatical? Please answer 1 or 2. - - {% if shuffled_order == 0 %} - - 1: {{ sentence_good }} - - 2: {{ sentence_bad }} - - {% else %} - - 1: {{ sentence_bad }} - - 2: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - 1 + b782d11e-a941-4ade-85af-93756a53297b: !Template + answer_choices: Yes ||| No + id: b782d11e-a941-4ade-85af-93756a53297b + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - {% else %} + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' + reference: '' + c1814309-3ce7-400e-8aef-94f465f2f0d8: !Template + answer_choices: Yes ||| No + id: c1814309-3ce7-400e-8aef-94f465f2f0d8 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - 2 + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' + reference: '' + de22063f-3ae4-442d-877f-d9326e90de27: !Template + answer_choices: Yes ||| No + id: de22063f-3ae4-442d-877f-d9326e90de27 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - {% endif %}' + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: random_choice_1_2_2 + original_task: false + name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' reference: '' diff --git a/promptsource/templates/blimp/transitive/templates.yaml b/promptsource/templates/blimp/transitive/templates.yaml index f5e3ad555..39a1ef564 100644 --- a/promptsource/templates/blimp/transitive/templates.yaml +++ b/promptsource/templates/blimp/transitive/templates.yaml @@ -1,9 +1,9 @@ dataset: blimp subset: transitive templates: - 0242c638-38a4-44e6-a5b4-b09039c3e9e1: !Template + 0481dc81-b5d6-4003-9f23-8b64a49d5296: !Template answer_choices: A ||| B - id: 0242c638-38a4-44e6-a5b4-b09039c3e9e1 + id: 0481dc81-b5d6-4003-9f23-8b64a49d5296 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -37,85 +37,124 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized + name: 'A/B choice randomized (choice order: A-B)' reference: '' - 259aab32-1ce2-42c3-bd9a-e4f8210fc784: !Template + 08a59f81-6b04-4f9a-bcfe-4a5dd9b2ba70: !Template + answer_choices: A ||| B + id: 08a59f81-6b04-4f9a-bcfe-4a5dd9b2ba70 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: 'A/B choice randomized (choice order: B-A)' + reference: '' + 20291886-1104-435d-af59-62f92208ab8a: !Template answer_choices: Yes ||| No - id: 259aab32-1ce2-42c3-bd9a-e4f8210fc784 + id: 20291886-1104-435d-af59-62f92208ab8a jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: Yes/No binary judgment for bad sentences + name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' reference: '' - 4f72a2dd-5108-4248-826c-2a86d56f62f7: !Template + 218a0707-57b1-42f4-9467-da6c3f839d12: !Template answer_choices: Yes ||| No - id: 4f72a2dd-5108-4248-826c-2a86d56f62f7 + id: 218a0707-57b1-42f4-9467-da6c3f839d12 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: Yes/No binary judgment for good sentences (choice order reversed) + name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' reference: '' - 4fa2f0ed-854a-41ce-b5c9-776910067e11: !Template + 3b12589b-4746-442b-a2e3-001f9e8ac024: !Template answer_choices: Yes ||| No - id: 4fa2f0ed-854a-41ce-b5c9-776910067e11 + id: 3b12589b-4746-442b-a2e3-001f9e8ac024 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: Yes/No binary judgment for bad sentences (choice order reversed) + name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' reference: '' - 58d8dba9-b98b-4772-b2fb-54837d03a32d: !Template - answer_choices: A ||| B - id: 58d8dba9-b98b-4772-b2fb-54837d03a32d + 3b247a68-a7ea-4330-ac9c-0b3c330fd2a2: !Template + answer_choices: null + id: 3b247a68-a7ea-4330-ac9c-0b3c330fd2a2 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| - {{ answer_choices[0] }} + {% if shuffled_order == 0 %} + + A {% else %} - {{ answer_choices[1] }} + B {% endif %}' metadata: !TemplateMetadata @@ -123,26 +162,26 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized (choice order reversed) + name: random_choice_A_B_2 reference: '' - 66bd9d77-b1f0-44fe-a4b1-4a6555ba2897: !Template + 4042fd6c-ea58-4f35-98ce-d6511fbf6428: !Template answer_choices: null - id: 66bd9d77-b1f0-44fe-a4b1-4a6555ba2897 + id: 4042fd6c-ea58-4f35-98ce-d6511fbf6428 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -150,11 +189,11 @@ templates: {% if shuffled_order == 0 %} - A + 1 {% else %} - B + 2 {% endif %}' metadata: !TemplateMetadata @@ -162,11 +201,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: random_choice_1_2 reference: '' - 9ee380ca-3ec7-4851-bf7d-d571b014edeb: !Template + 4c3ea778-4682-43f5-9169-986a1127e8a2: !Template answer_choices: null - id: 9ee380ca-3ec7-4851-bf7d-d571b014edeb + id: 4c3ea778-4682-43f5-9169-986a1127e8a2 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer A or B. @@ -203,9 +242,9 @@ templates: original_task: true name: random_choice_A_B reference: '' - c1f4ab12-6569-4495-86f4-bbc5552426a4: !Template + b8342f4e-ddff-44d4-a153-4515e4bf381f: !Template answer_choices: null - id: c1f4ab12-6569-4495-86f4-bbc5552426a4 + id: b8342f4e-ddff-44d4-a153-4515e4bf381f jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -242,58 +281,19 @@ templates: original_task: true name: random_choice_1_2_2 reference: '' - ca678159-1031-4ffa-ab54-3953f80e70ba: !Template + e6ab77e5-bca0-430a-93d4-6973b63a221a: !Template answer_choices: Yes ||| No - id: ca678159-1031-4ffa-ab54-3953f80e70ba + id: e6ab77e5-bca0-430a-93d4-6973b63a221a jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: Yes/No binary judgment for good sentences - reference: '' - e3f3a6c4-34b3-4bcc-995d-53e1724e37b7: !Template - answer_choices: null - id: e3f3a6c4-34b3-4bcc-995d-53e1724e37b7 - jinja: '{% set shuffled_order = [0, 1] | random %} - - Which one of the following sentences is grammatical? Please answer 1 or 2. - - {% if shuffled_order == 0 %} - - 1: {{ sentence_good }} - - 2: {{ sentence_bad }} - - {% else %} - - 1: {{ sentence_bad }} - - 2: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - 1 - - {% else %} - - 2 - - {% endif %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: random_choice_1_2 + name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' reference: '' diff --git a/promptsource/templates/blimp/wh_island/templates.yaml b/promptsource/templates/blimp/wh_island/templates.yaml index 0e64fa92c..e76af84e6 100644 --- a/promptsource/templates/blimp/wh_island/templates.yaml +++ b/promptsource/templates/blimp/wh_island/templates.yaml @@ -1,24 +1,40 @@ dataset: blimp subset: wh_island templates: - 2dca157a-1b3b-4d17-9a66-d2f9ed4cda69: !Template + 04f03f62-9cf6-46cd-a4e9-e7b48e240825: !Template + answer_choices: Yes ||| No + id: 04f03f62-9cf6-46cd-a4e9-e7b48e240825 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' + reference: '' + 0ba65f3a-fe83-4b11-9840-e8d7f25c0fad: !Template answer_choices: null - id: 2dca157a-1b3b-4d17-9a66-d2f9ed4cda69 + id: 0ba65f3a-fe83-4b11-9840-e8d7f25c0fad jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -26,11 +42,11 @@ templates: {% if shuffled_order == 0 %} - 1 + A {% else %} - 2 + B {% endif %}' metadata: !TemplateMetadata @@ -38,27 +54,49 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: random_choice_A_B reference: '' - 409916e0-3857-4920-9494-05f49404f55a: !Template - answer_choices: Yes ||| No - id: 409916e0-3857-4920-9494-05f49404f55a - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + 0dba9bb2-8fad-4aa0-9c4f-269050428da8: !Template + answer_choices: A ||| B + id: 0dba9bb2-8fad-4aa0-9c4f-269050428da8 + jinja: '{% set shuffled_order = [0, 1] | random %} - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences + original_task: true + name: 'A/B choice randomized (choice order: A-B)' reference: '' - 4f6f5f6f-fbe2-425c-8507-b753ca2beccd: !Template + 32562d86-9a21-4c06-9ae5-9e080e53bd04: !Template answer_choices: null - id: 4f6f5f6f-fbe2-425c-8507-b753ca2beccd + id: 32562d86-9a21-4c06-9ae5-9e080e53bd04 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -95,25 +133,9 @@ templates: original_task: true name: random_choice_1_2_2 reference: '' - 60d5c928-36e2-42ab-87f1-b0335a5ec8a9: !Template - answer_choices: Yes ||| No - id: 60d5c928-36e2-42ab-87f1-b0335a5ec8a9 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences - reference: '' - 6131e963-2b15-4038-a787-e6fdff231165: !Template + 3cf67ba6-3dc6-4c76-afb4-f0b7dd5a1d78: !Template answer_choices: A ||| B - id: 6131e963-2b15-4038-a787-e6fdff231165 + id: 3cf67ba6-3dc6-4c76-afb4-f0b7dd5a1d78 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -147,26 +169,26 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized (choice order reversed) + name: 'A/B choice randomized (choice order: B-A)' reference: '' - 62226dae-1362-499e-b419-e756d455358f: !Template + 5447fec4-045c-4578-a120-749f4c308462: !Template answer_choices: null - id: 62226dae-1362-499e-b419-e756d455358f + id: 5447fec4-045c-4578-a120-749f4c308462 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -174,11 +196,11 @@ templates: {% if shuffled_order == 0 %} - A + 1 {% else %} - B + 2 {% endif %}' metadata: !TemplateMetadata @@ -186,11 +208,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: random_choice_1_2 reference: '' - a2e35198-bf39-45d7-ba76-6bcda9454402: !Template + c43490fa-7c7e-415b-ad5d-f449b571b261: !Template answer_choices: Yes ||| No - id: a2e35198-bf39-45d7-ba76-6bcda9454402 + id: c43490fa-7c7e-415b-ad5d-f449b571b261 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -202,27 +224,11 @@ templates: metrics: - Accuracy original_task: false - name: Yes/No binary judgment for good sentences (choice order reversed) + name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' reference: '' - c6540e39-22ab-4609-bd50-380ac577edf3: !Template - answer_choices: Yes ||| No - id: c6540e39-22ab-4609-bd50-380ac577edf3 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences (choice order reversed) - reference: '' - d9f8e6db-52bc-4d96-9822-4e93490771cd: !Template + c56d413e-ac51-412c-a2d3-11d66a3559b3: !Template answer_choices: null - id: d9f8e6db-52bc-4d96-9822-4e93490771cd + id: c56d413e-ac51-412c-a2d3-11d66a3559b3 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer A or B. @@ -259,41 +265,35 @@ templates: original_task: true name: random_choice_A_B_2 reference: '' - f940fb16-8b1a-4956-b992-e18e3a3a0ada: !Template - answer_choices: A ||| B - id: f940fb16-8b1a-4956-b992-e18e3a3a0ada - jinja: '{% set shuffled_order = [0, 1] | random %} - - Read the following two sentences. - - {% if shuffled_order == 0 %} - - Sentence A: {{ sentence_good }} - - Sentence B: {{ sentence_bad }} - - {% else %} - - Sentence A: {{ sentence_bad }} - - Sentence B: {{ sentence_good }} - - {% endif %} - - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} - - {{ answer_choices[0] }} - - {% else %} + d7146feb-1a26-44ae-af5a-00595cf9f210: !Template + answer_choices: Yes ||| No + id: d7146feb-1a26-44ae-af5a-00595cf9f210 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - {{ answer_choices[1] }} + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' + reference: '' + e7be32ff-90ce-4472-9917-4075baa411a7: !Template + answer_choices: Yes ||| No + id: e7be32ff-90ce-4472-9917-4075baa411a7 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - {% endif %}' + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: A/B choice randomized + original_task: false + name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' reference: '' diff --git a/promptsource/templates/blimp/wh_questions_object_gap/templates.yaml b/promptsource/templates/blimp/wh_questions_object_gap/templates.yaml index 5f65d3804..d23e4646f 100644 --- a/promptsource/templates/blimp/wh_questions_object_gap/templates.yaml +++ b/promptsource/templates/blimp/wh_questions_object_gap/templates.yaml @@ -1,67 +1,44 @@ dataset: blimp subset: wh_questions_object_gap templates: - 0d855a12-4a18-47bd-88d1-700c0e93332a: !Template - answer_choices: null - id: 0d855a12-4a18-47bd-88d1-700c0e93332a - jinja: '{% set shuffled_order = [0, 1] | random %} - - Between the two sentences, which one is grammatical? Please answer A or B. - - {% if shuffled_order == 0 %} - - A: {{ sentence_good }} - - B: {{ sentence_bad }} - - {% else %} - - A: {{ sentence_bad }} - - B: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - A - - {% else %} - - B + 057ce505-36bc-4c6a-873f-7ead936f3a33: !Template + answer_choices: Yes ||| No + id: 057ce505-36bc-4c6a-873f-7ead936f3a33 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - {% endif %}' + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: random_choice_A_B_2 + original_task: false + name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' reference: '' - 10b23d23-7952-47f5-b525-c87c1e015103: !Template + 0cd48e77-78d8-4c47-b051-9890e583eda6: !Template answer_choices: Yes ||| No - id: 10b23d23-7952-47f5-b525-c87c1e015103 + id: 0cd48e77-78d8-4c47-b051-9890e583eda6 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: Yes/No binary judgment for bad sentences + name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' reference: '' - 129d8ec0-b4f0-4ef4-8333-d7d8985a2f5e: !Template + 3512ec57-5b8c-4a06-b375-173096c697d6: !Template answer_choices: null - id: 129d8ec0-b4f0-4ef4-8333-d7d8985a2f5e + id: 3512ec57-5b8c-4a06-b375-173096c697d6 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -93,14 +70,14 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: random_choice_A_B_2 reference: '' - 132135cb-07e3-4615-ab43-16adc300b9c9: !Template + 5c17a440-b0ff-4d27-b6d1-535b937f6348: !Template answer_choices: null - id: 132135cb-07e3-4615-ab43-16adc300b9c9 + id: 5c17a440-b0ff-4d27-b6d1-535b937f6348 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -132,11 +109,27 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: random_choice_1_2_2 reference: '' - 4d3c0d5c-996c-4b1c-bc6e-649cb1b4d35a: !Template + 5e9c971d-520b-4e0b-8442-e0bb719f40ec: !Template + answer_choices: Yes ||| No + id: 5e9c971d-520b-4e0b-8442-e0bb719f40ec + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' + reference: '' + b29436b6-37e5-4915-a91d-991c61c30951: !Template answer_choices: A ||| B - id: 4d3c0d5c-996c-4b1c-bc6e-649cb1b4d35a + id: b29436b6-37e5-4915-a91d-991c61c30951 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -155,7 +148,7 @@ templates: {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order + Which one is a better sentence of English, B or A? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -170,42 +163,26 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized - reference: '' - 54cd4dae-b2f3-4bdc-bcc8-5400a7c4bb6a: !Template - answer_choices: Yes ||| No - id: 54cd4dae-b2f3-4bdc-bcc8-5400a7c4bb6a - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences (choice order reversed) + name: 'A/B choice randomized (choice order: B-A)' reference: '' - 6d9efb60-2940-494e-8b6e-dc1961a3cd5b: !Template + c2dbc938-a331-4127-b66c-3d3be7074239: !Template answer_choices: null - id: 6d9efb60-2940-494e-8b6e-dc1961a3cd5b + id: c2dbc938-a331-4127-b66c-3d3be7074239 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -213,11 +190,11 @@ templates: {% if shuffled_order == 0 %} - 1 + A {% else %} - 2 + B {% endif %}' metadata: !TemplateMetadata @@ -225,27 +202,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 - reference: '' - cc83c2b5-75c2-45b7-a077-16d7ba336c29: !Template - answer_choices: Yes ||| No - id: cc83c2b5-75c2-45b7-a077-16d7ba336c29 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences + name: random_choice_A_B reference: '' - e0608d5d-5605-49b2-b9b2-68717a53f2c9: !Template + de5a8942-bc5a-4138-912f-142eef1f11f2: !Template answer_choices: A ||| B - id: e0608d5d-5605-49b2-b9b2-68717a53f2c9 + id: de5a8942-bc5a-4138-912f-142eef1f11f2 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -264,7 +225,7 @@ templates: {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order + Which one is a better sentence of English, A or B? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -279,21 +240,60 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized (choice order reversed) + name: 'A/B choice randomized (choice order: A-B)' reference: '' - f47e01a4-1c90-4d01-b085-c0655cdedc46: !Template + e652d0f3-a32f-4f31-ba83-afd6408fa40f: !Template answer_choices: Yes ||| No - id: f47e01a4-1c90-4d01-b085-c0655cdedc46 + id: e652d0f3-a32f-4f31-ba83-afd6408fa40f jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: Yes/No binary judgment for good sentences (choice order reversed) + name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' + reference: '' + fc3c593f-7bea-43e1-96ad-ef1fa8a7fb16: !Template + answer_choices: null + id: fc3c593f-7bea-43e1-96ad-ef1fa8a7fb16 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2 reference: '' diff --git a/promptsource/templates/blimp/wh_questions_subject_gap/templates.yaml b/promptsource/templates/blimp/wh_questions_subject_gap/templates.yaml index efdf566a8..0dfdbf594 100644 --- a/promptsource/templates/blimp/wh_questions_subject_gap/templates.yaml +++ b/promptsource/templates/blimp/wh_questions_subject_gap/templates.yaml @@ -1,36 +1,35 @@ dataset: blimp subset: wh_questions_subject_gap templates: - 10166d03-6bdd-4eb9-a621-686310e7dbc3: !Template - answer_choices: null - id: 10166d03-6bdd-4eb9-a621-686310e7dbc3 + 0dfdb9b5-374e-443f-82c2-3498e7ef10ed: !Template + answer_choices: A ||| B + id: 0dfdb9b5-374e-443f-82c2-3498e7ef10ed jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -38,50 +37,27 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: 'A/B choice randomized (choice order: B-A)' reference: '' - 37339aa3-e09b-4c0f-9831-f2aff9bcf11c: !Template - answer_choices: null - id: 37339aa3-e09b-4c0f-9831-f2aff9bcf11c - jinja: '{% set shuffled_order = [0, 1] | random %} - - Which one of the following sentences is grammatical? Please answer A or B. - - {% if shuffled_order == 0 %} - - A: {{ sentence_good }} - - B: {{ sentence_bad }} - - {% else %} - - A: {{ sentence_bad }} - - B: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - A - - {% else %} - - B + 1df0cb41-8c9e-4e84-8146-c6f8a586e7fc: !Template + answer_choices: Yes ||| No + id: 1df0cb41-8c9e-4e84-8146-c6f8a586e7fc + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - {% endif %}' + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: random_choice_A_B + original_task: false + name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' reference: '' - 4d0a99ab-3986-4e8f-8918-b8c4bbf0f4f7: !Template + 36e5bebe-4cf7-47db-890d-c53fff0e1e09: !Template answer_choices: Yes ||| No - id: 4d0a99ab-3986-4e8f-8918-b8c4bbf0f4f7 + id: 36e5bebe-4cf7-47db-890d-c53fff0e1e09 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -93,14 +69,14 @@ templates: metrics: - Accuracy original_task: false - name: Yes/No binary judgment for bad sentences (choice order reversed) + name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' reference: '' - 8a0bf278-a9d2-4940-9215-7d3e30e18280: !Template + 53bb4f17-cc2c-40ee-b696-77500fcff9eb: !Template answer_choices: null - id: 8a0bf278-a9d2-4940-9215-7d3e30e18280 + id: 53bb4f17-cc2c-40ee-b696-77500fcff9eb jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -132,37 +108,38 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: random_choice_1_2_2 reference: '' - a1c77e55-42a2-4ea7-ab7b-aa967e5b5c76: !Template - answer_choices: A ||| B - id: a1c77e55-42a2-4ea7-ab7b-aa967e5b5c76 + 6ee20533-4e94-4fe3-bc31-d9664905faeb: !Template + answer_choices: null + id: 6ee20533-4e94-4fe3-bc31-d9664905faeb jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| - {{ answer_choices[0] }} + {% if shuffled_order == 0 %} + + A {% else %} - {{ answer_choices[1] }} + B {% endif %}' metadata: !TemplateMetadata @@ -170,69 +147,70 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized + name: random_choice_A_B_2 reference: '' - aa2f941c-3501-42fd-9f68-8a20d23a25b6: !Template + a694e34c-2731-492d-ab18-63575ec8befb: !Template answer_choices: Yes ||| No - id: aa2f941c-3501-42fd-9f68-8a20d23a25b6 + id: a694e34c-2731-492d-ab18-63575ec8befb jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: Yes/No binary judgment for bad sentences + name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' reference: '' - b734faa8-895b-44f8-9426-fa45e8664045: !Template + d53c294d-ca94-4e3c-97d6-917acada0754: !Template answer_choices: Yes ||| No - id: b734faa8-895b-44f8-9426-fa45e8664045 + id: d53c294d-ca94-4e3c-97d6-917acada0754 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: Yes/No binary judgment for good sentences + name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' reference: '' - b982dab8-15b0-4554-94e6-fad6a1d7e490: !Template - answer_choices: A ||| B - id: b982dab8-15b0-4554-94e6-fad6a1d7e490 + dd4a46eb-ffd5-4e9a-8068-245ff68a3e7b: !Template + answer_choices: null + id: dd4a46eb-ffd5-4e9a-8068-245ff68a3e7b jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| - {{ answer_choices[0] }} + {% if shuffled_order == 0 %} + + A {% else %} - {{ answer_choices[1] }} + B {% endif %}' metadata: !TemplateMetadata @@ -240,54 +218,76 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized (choice order reversed) + name: random_choice_A_B reference: '' - c14abdc3-d9a6-4a7e-9e85-698fc20067dd: !Template - answer_choices: Yes ||| No - id: c14abdc3-d9a6-4a7e-9e85-698fc20067dd - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + f0179b4a-9fdb-430c-928c-8accc7ba42cf: !Template + answer_choices: null + id: f0179b4a-9fdb-430c-928c-8accc7ba42cf + jinja: '{% set shuffled_order = [0, 1] | random %} - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences (choice order reversed) + original_task: true + name: random_choice_1_2 reference: '' - fd661930-125a-4a9a-97d7-bbab9ba1fd1b: !Template - answer_choices: null - id: fd661930-125a-4a9a-97d7-bbab9ba1fd1b + f9cb8868-cfea-4558-9e94-c66923e3b110: !Template + answer_choices: A ||| B + id: f9cb8868-cfea-4558-9e94-c66923e3b110 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -295,5 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: 'A/B choice randomized (choice order: A-B)' reference: '' diff --git a/promptsource/templates/blimp/wh_questions_subject_gap_long_distance/templates.yaml b/promptsource/templates/blimp/wh_questions_subject_gap_long_distance/templates.yaml index 9e2aaf2ef..3eb4a2ebe 100644 --- a/promptsource/templates/blimp/wh_questions_subject_gap_long_distance/templates.yaml +++ b/promptsource/templates/blimp/wh_questions_subject_gap_long_distance/templates.yaml @@ -1,35 +1,36 @@ dataset: blimp subset: wh_questions_subject_gap_long_distance templates: - 02997aad-8b15-4ce2-96b1-57af313e5b6a: !Template - answer_choices: A ||| B - id: 02997aad-8b15-4ce2-96b1-57af313e5b6a + 4a4adcfb-18f9-4ad0-8272-ddd5ce2c02dd: !Template + answer_choices: null + id: 4a4adcfb-18f9-4ad0-8272-ddd5ce2c02dd jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| - {{ answer_choices[0] }} + {% if shuffled_order == 0 %} + + A {% else %} - {{ answer_choices[1] }} + B {% endif %}' metadata: !TemplateMetadata @@ -37,37 +38,38 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized (choice order reversed) + name: random_choice_A_B reference: '' - 0ad26bac-7501-448c-b702-0cd0c05781e5: !Template - answer_choices: A ||| B - id: 0ad26bac-7501-448c-b702-0cd0c05781e5 + 4d15b6c5-c750-4a18-9c92-f9d339e4e78d: !Template + answer_choices: null + id: 4d15b6c5-c750-4a18-9c92-f9d339e4e78d jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| - {{ answer_choices[0] }} + {% if shuffled_order == 0 %} + + A {% else %} - {{ answer_choices[1] }} + B {% endif %}' metadata: !TemplateMetadata @@ -75,54 +77,37 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized - reference: '' - 17b23ee6-9f95-4fcc-b8d1-65583413879e: !Template - answer_choices: Yes ||| No - id: 17b23ee6-9f95-4fcc-b8d1-65583413879e - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences + name: random_choice_A_B_2 reference: '' - 517d74d1-17e3-404c-b7d3-99a91ddb34ff: !Template - answer_choices: null - id: 517d74d1-17e3-404c-b7d3-99a91ddb34ff + 56196cd7-6e97-464b-bb60-11e901c32f42: !Template + answer_choices: A ||| B + id: 56196cd7-6e97-464b-bb60-11e901c32f42 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -130,26 +115,26 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: 'A/B choice randomized (choice order: A-B)' reference: '' - 77ba60bb-28cd-4d3e-8a42-53c0bf276a33: !Template + 5c73149e-6698-40dd-b76b-2ec5da7cfa2b: !Template answer_choices: null - id: 77ba60bb-28cd-4d3e-8a42-53c0bf276a33 + id: 5c73149e-6698-40dd-b76b-2ec5da7cfa2b jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -157,11 +142,11 @@ templates: {% if shuffled_order == 0 %} - A + 1 {% else %} - B + 2 {% endif %}' metadata: !TemplateMetadata @@ -169,11 +154,59 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: random_choice_1_2_2 + reference: '' + a7b5726a-9c85-4a67-9064-7510093db2bf: !Template + answer_choices: Yes ||| No + id: a7b5726a-9c85-4a67-9064-7510093db2bf + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' + reference: '' + c4e3a884-1219-4fd1-92cc-1da52d48b934: !Template + answer_choices: Yes ||| No + id: c4e3a884-1219-4fd1-92cc-1da52d48b934 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' reference: '' - 7cd3851b-8a83-408d-b8b5-da92dd0eb653: !Template + d2ca0f0e-586f-4476-a1c0-41b1a2faf47a: !Template + answer_choices: Yes ||| No + id: d2ca0f0e-586f-4476-a1c0-41b1a2faf47a + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' + reference: '' + d39e16be-1272-4358-8ab1-3bb4e80bc87b: !Template answer_choices: null - id: 7cd3851b-8a83-408d-b8b5-da92dd0eb653 + id: d39e16be-1272-4358-8ab1-3bb4e80bc87b jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer 1 or 2. @@ -210,36 +243,35 @@ templates: original_task: true name: random_choice_1_2 reference: '' - 83ea467e-deda-408e-9d65-5e56c8a88614: !Template - answer_choices: null - id: 83ea467e-deda-408e-9d65-5e56c8a88614 + d7692403-42a6-4b12-bea7-13b3abf271c0: !Template + answer_choices: A ||| B + id: d7692403-42a6-4b12-bea7-13b3abf271c0 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -247,47 +279,15 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 - reference: '' - a6acbe8c-93ed-43c4-92da-846c2e33b17e: !Template - answer_choices: Yes ||| No - id: a6acbe8c-93ed-43c4-92da-846c2e33b17e - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences + name: 'A/B choice randomized (choice order: B-A)' reference: '' - cc00df16-e40a-47f0-b290-b891621b173a: !Template + e07ea319-9726-4e78-ac86-aa18cb45ade4: !Template answer_choices: Yes ||| No - id: cc00df16-e40a-47f0-b290-b891621b173a + id: e07ea319-9726-4e78-ac86-aa18cb45ade4 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences (choice order reversed) - reference: '' - db0dfb85-8fca-4470-8582-0842152a01b5: !Template - answer_choices: Yes ||| No - id: db0dfb85-8fca-4470-8582-0842152a01b5 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata @@ -295,5 +295,5 @@ templates: metrics: - Accuracy original_task: false - name: Yes/No binary judgment for bad sentences (choice order reversed) + name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' reference: '' diff --git a/promptsource/templates/blimp/wh_vs_that_no_gap/templates.yaml b/promptsource/templates/blimp/wh_vs_that_no_gap/templates.yaml index 7f664b9ac..6cd1a4c1d 100644 --- a/promptsource/templates/blimp/wh_vs_that_no_gap/templates.yaml +++ b/promptsource/templates/blimp/wh_vs_that_no_gap/templates.yaml @@ -1,9 +1,25 @@ dataset: blimp subset: wh_vs_that_no_gap templates: - 2caf6cd8-7580-44a0-adb1-eac711d23513: !Template + 08741ec2-b18b-48ea-9093-a7abdfe263e1: !Template + answer_choices: Yes ||| No + id: 08741ec2-b18b-48ea-9093-a7abdfe263e1 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' + reference: '' + 126eb196-b508-4ec9-9b0a-febb0896d683: !Template answer_choices: null - id: 2caf6cd8-7580-44a0-adb1-eac711d23513 + id: 126eb196-b508-4ec9-9b0a-febb0896d683 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -40,25 +56,9 @@ templates: original_task: true name: random_choice_1_2_2 reference: '' - 6bd58b9b-9d0b-4856-a5c8-92ab4883d700: !Template - answer_choices: Yes ||| No - id: 6bd58b9b-9d0b-4856-a5c8-92ab4883d700 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences (choice order reversed) - reference: '' - 83f6a959-8a8d-47c7-a78d-e0f10ed8628f: !Template + 598ac8e0-c837-4394-9e63-7aae2b17a742: !Template answer_choices: A ||| B - id: 83f6a959-8a8d-47c7-a78d-e0f10ed8628f + id: 598ac8e0-c837-4394-9e63-7aae2b17a742 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -92,43 +92,49 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized + name: 'A/B choice randomized (choice order: A-B)' reference: '' - addc8a1c-1c2a-4a76-9a2a-0eb65701ba5a: !Template - answer_choices: Yes ||| No - id: addc8a1c-1c2a-4a76-9a2a-0eb65701ba5a - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + 83ee5881-9a7d-430b-ac8f-cd4b425729ad: !Template + answer_choices: A ||| B + id: 83ee5881-9a7d-430b-ac8f-cd4b425729ad + jinja: '{% set shuffled_order = [0, 1] | random %} - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences (choice order reversed) - reference: '' - c1086e63-290c-4ccb-a533-22e37825bfcb: !Template - answer_choices: Yes ||| No - id: c1086e63-290c-4ccb-a533-22e37825bfcb - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + Read the following two sentences. - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences + original_task: true + name: 'A/B choice randomized (choice order: B-A)' reference: '' - c8bf7a4d-2415-43c8-88be-6835ee6f9d4e: !Template + 90e9e76b-791a-4c5e-89b6-ffb13061ca98: !Template answer_choices: null - id: c8bf7a4d-2415-43c8-88be-6835ee6f9d4e + id: 90e9e76b-791a-4c5e-89b6-ffb13061ca98 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer 1 or 2. @@ -165,9 +171,9 @@ templates: original_task: true name: random_choice_1_2 reference: '' - ce494cf7-6df6-4ec2-b329-5d7d5d8ba2bc: !Template + 9fed0c48-2413-499f-9b7b-b42d3a425d92: !Template answer_choices: Yes ||| No - id: ce494cf7-6df6-4ec2-b329-5d7d5d8ba2bc + id: 9fed0c48-2413-499f-9b7b-b42d3a425d92 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -179,11 +185,11 @@ templates: metrics: - Accuracy original_task: false - name: Yes/No binary judgment for bad sentences + name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' reference: '' - e529f7d4-ebbf-4240-9616-01b3761b5bcd: !Template + bd3ed699-075b-40f1-9b8e-bb28179645dd: !Template answer_choices: null - id: e529f7d4-ebbf-4240-9616-01b3761b5bcd + id: bd3ed699-075b-40f1-9b8e-bb28179645dd jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer A or B. @@ -220,47 +226,9 @@ templates: original_task: true name: random_choice_A_B_2 reference: '' - ef36d150-f6d0-4375-b6bc-6b8339a0a8bc: !Template - answer_choices: A ||| B - id: ef36d150-f6d0-4375-b6bc-6b8339a0a8bc - jinja: '{% set shuffled_order = [0, 1] | random %} - - Read the following two sentences. - - {% if shuffled_order == 0 %} - - Sentence A: {{ sentence_good }} - - Sentence B: {{ sentence_bad }} - - {% else %} - - Sentence A: {{ sentence_bad }} - - Sentence B: {{ sentence_good }} - - {% endif %} - - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} - - {% endif %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: A/B choice randomized (choice order reversed) - reference: '' - f9f889b4-4789-4529-9792-ef09b3b2544c: !Template + bdb81024-b38f-441d-9533-96ea05dc9faa: !Template answer_choices: null - id: f9f889b4-4789-4529-9792-ef09b3b2544c + id: bdb81024-b38f-441d-9533-96ea05dc9faa jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer A or B. @@ -297,3 +265,35 @@ templates: original_task: true name: random_choice_A_B reference: '' + d3399774-0cec-4c85-8348-2a78eff8508e: !Template + answer_choices: Yes ||| No + id: d3399774-0cec-4c85-8348-2a78eff8508e + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' + reference: '' + eb12dd1f-efcd-4835-aaff-96fe44bab9b2: !Template + answer_choices: Yes ||| No + id: eb12dd1f-efcd-4835-aaff-96fe44bab9b2 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' + reference: '' diff --git a/promptsource/templates/blimp/wh_vs_that_no_gap_long_distance/templates.yaml b/promptsource/templates/blimp/wh_vs_that_no_gap_long_distance/templates.yaml index eeb3b0a6e..73bb92cc9 100644 --- a/promptsource/templates/blimp/wh_vs_that_no_gap_long_distance/templates.yaml +++ b/promptsource/templates/blimp/wh_vs_that_no_gap_long_distance/templates.yaml @@ -1,62 +1,24 @@ dataset: blimp subset: wh_vs_that_no_gap_long_distance templates: - 0d2df529-7e58-4195-ac23-401595e22fa7: !Template - answer_choices: A ||| B - id: 0d2df529-7e58-4195-ac23-401595e22fa7 - jinja: '{% set shuffled_order = [0, 1] | random %} - - Read the following two sentences. - - {% if shuffled_order == 0 %} - - Sentence A: {{ sentence_good }} - - Sentence B: {{ sentence_bad }} - - {% else %} - - Sentence A: {{ sentence_bad }} - - Sentence B: {{ sentence_good }} - - {% endif %} - - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} - - {% endif %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: A/B choice randomized (choice order reversed) - reference: '' - 2e268d7a-96e2-4bce-a6d6-a84d42bc7661: !Template + 24d4f457-c24e-462d-a5eb-9cce58b17b1b: !Template answer_choices: null - id: 2e268d7a-96e2-4bce-a6d6-a84d42bc7661 + id: 24d4f457-c24e-462d-a5eb-9cce58b17b1b jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -64,11 +26,11 @@ templates: {% if shuffled_order == 0 %} - A + 1 {% else %} - B + 2 {% endif %}' metadata: !TemplateMetadata @@ -76,38 +38,37 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: random_choice_1_2_2 reference: '' - 5b1f54d8-5325-4314-bfcd-518753e52762: !Template - answer_choices: null - id: 5b1f54d8-5325-4314-bfcd-518753e52762 + 261d37b8-c0f5-4ab3-bad6-b914fab9f748: !Template + answer_choices: A ||| B + id: 261d37b8-c0f5-4ab3-bad6-b914fab9f748 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -115,31 +76,31 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: 'A/B choice randomized (choice order: B-A)' reference: '' - 68845ea6-10ad-400b-b680-a1aa576a1059: !Template + 2da6237a-18fe-4f35-837b-983bb0bd8ecb: !Template answer_choices: Yes ||| No - id: 68845ea6-10ad-400b-b680-a1aa576a1059 + id: 2da6237a-18fe-4f35-837b-983bb0bd8ecb jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: Yes/No binary judgment for good sentences (choice order reversed) + name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' reference: '' - 80f08dc9-709a-4a1c-a741-aba516eb3bb3: !Template + 338fb184-4c96-4557-9916-0989d2e71027: !Template answer_choices: Yes ||| No - id: 80f08dc9-709a-4a1c-a741-aba516eb3bb3 + id: 338fb184-4c96-4557-9916-0989d2e71027 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -147,42 +108,42 @@ templates: metrics: - Accuracy original_task: false - name: Yes/No binary judgment for good sentences + name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' reference: '' - 94016858-104f-4181-b395-d7ed6740b4a4: !Template + 4b0e8f53-6e0f-402c-93f4-d59a44593a47: !Template answer_choices: Yes ||| No - id: 94016858-104f-4181-b395-d7ed6740b4a4 + id: 4b0e8f53-6e0f-402c-93f4-d59a44593a47 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: Yes/No binary judgment for bad sentences (choice order reversed) + name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' reference: '' - 94282ca6-d70a-452f-a8d3-24d12d5b6e41: !Template + 76821051-fc90-4b78-8717-f886ddded5da: !Template answer_choices: null - id: 94282ca6-d70a-452f-a8d3-24d12d5b6e41 + id: 76821051-fc90-4b78-8717-f886ddded5da jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -190,11 +151,11 @@ templates: {% if shuffled_order == 0 %} - 1 + A {% else %} - 2 + B {% endif %}' metadata: !TemplateMetadata @@ -202,11 +163,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: random_choice_A_B reference: '' - 9560095e-3b90-450a-a0b2-a18b6e9659a8: !Template + 9b6e51cc-079f-44ee-bce9-7558da58e11f: !Template answer_choices: Yes ||| No - id: 9560095e-3b90-450a-a0b2-a18b6e9659a8 + id: 9b6e51cc-079f-44ee-bce9-7558da58e11f jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -218,11 +179,11 @@ templates: metrics: - Accuracy original_task: false - name: Yes/No binary judgment for bad sentences + name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' reference: '' - afe6553a-98e8-4b0a-b59c-91e1705fa192: !Template + aec7db7f-1eb7-4247-ac42-4c3a429f36aa: !Template answer_choices: A ||| B - id: afe6553a-98e8-4b0a-b59c-91e1705fa192 + id: aec7db7f-1eb7-4247-ac42-4c3a429f36aa jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -256,14 +217,14 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized + name: 'A/B choice randomized (choice order: A-B)' reference: '' - e06fd840-fb46-428a-855c-554c50e5645e: !Template + d36cfd2a-8c25-4c74-80ee-5bc90a6f7b00: !Template answer_choices: null - id: e06fd840-fb46-428a-855c-554c50e5645e + id: d36cfd2a-8c25-4c74-80ee-5bc90a6f7b00 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -295,5 +256,44 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: random_choice_A_B_2 + reference: '' + e7ad406a-164c-4662-8341-b9febd044954: !Template + answer_choices: null + id: e7ad406a-164c-4662-8341-b9febd044954 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + 1 + + {% else %} + + 2 + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: random_choice_1_2 reference: '' diff --git a/promptsource/templates/blimp/wh_vs_that_with_gap/templates.yaml b/promptsource/templates/blimp/wh_vs_that_with_gap/templates.yaml index 84b6abb47..a8dacbace 100644 --- a/promptsource/templates/blimp/wh_vs_that_with_gap/templates.yaml +++ b/promptsource/templates/blimp/wh_vs_that_with_gap/templates.yaml @@ -1,40 +1,40 @@ dataset: blimp subset: wh_vs_that_with_gap templates: - 5caeb435-b9b0-48c5-a7a1-16cad62d65f3: !Template + 30d16980-86aa-47da-999d-eea0f4808ebb: !Template answer_choices: Yes ||| No - id: 5caeb435-b9b0-48c5-a7a1-16cad62d65f3 + id: 30d16980-86aa-47da-999d-eea0f4808ebb jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: Yes/No binary judgment for bad sentences (choice order reversed) + name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' reference: '' - 6477dd7a-0af8-4142-9d72-7ec0fdf54b1d: !Template + 88cfbc9a-7bd8-4a26-aaeb-68216eab7628: !Template answer_choices: null - id: 6477dd7a-0af8-4142-9d72-7ec0fdf54b1d + id: 88cfbc9a-7bd8-4a26-aaeb-68216eab7628 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -42,11 +42,11 @@ templates: {% if shuffled_order == 0 %} - 1 + A {% else %} - 2 + B {% endif %}' metadata: !TemplateMetadata @@ -54,43 +54,66 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: random_choice_A_B reference: '' - b973374b-1ba6-4472-8552-f458844c7f53: !Template + 9200c8a2-206d-4682-8d83-91f1b7e5c180: !Template answer_choices: Yes ||| No - id: b973374b-1ba6-4472-8552-f458844c7f53 + id: 9200c8a2-206d-4682-8d83-91f1b7e5c180 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: Yes/No binary judgment for good sentences (choice order reversed) + name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' reference: '' - be01a6d1-3bdf-4eb8-81d4-40e6bb8d8e9d: !Template - answer_choices: Yes ||| No - id: be01a6d1-3bdf-4eb8-81d4-40e6bb8d8e9d - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + b6f46e80-e359-48b1-ae2e-66b1385683f5: !Template + answer_choices: null + id: b6f46e80-e359-48b1-ae2e-66b1385683f5 + jinja: '{% set shuffled_order = [0, 1] | random %} - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + A + + {% else %} + + B + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences + original_task: true + name: random_choice_A_B_2 reference: '' - bff82d91-3194-48eb-ab61-3f5a4a53fdf5: !Template + c4397157-910e-493c-9482-55996ccd712c: !Template answer_choices: A ||| B - id: bff82d91-3194-48eb-ab61-3f5a4a53fdf5 + id: c4397157-910e-493c-9482-55996ccd712c jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -109,7 +132,7 @@ templates: {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order + Which one is a better sentence of English, B or A? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -124,54 +147,53 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized + name: 'A/B choice randomized (choice order: B-A)' reference: '' - c9da5037-e20a-44f8-94bc-df3c3014df13: !Template + c9b845cf-7b73-430c-81f4-c27286d6e2a0: !Template answer_choices: Yes ||| No - id: c9da5037-e20a-44f8-94bc-df3c3014df13 + id: c9b845cf-7b73-430c-81f4-c27286d6e2a0 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: Yes/No binary judgment for good sentences + name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' reference: '' - d8a2fd2f-9658-46eb-bc34-74fa8c5c17e0: !Template - answer_choices: null - id: d8a2fd2f-9658-46eb-bc34-74fa8c5c17e0 + cde97831-7118-4cdd-a28e-b369cb1cc6a0: !Template + answer_choices: A ||| B + id: cde97831-7118-4cdd-a28e-b369cb1cc6a0 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -179,26 +201,26 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: 'A/B choice randomized (choice order: A-B)' reference: '' - e6a4fc36-9684-44a0-83d0-d70a12c51ba1: !Template + d0dc04f5-e132-47f0-b264-ee44815a417b: !Template answer_choices: null - id: e6a4fc36-9684-44a0-83d0-d70a12c51ba1 + id: d0dc04f5-e132-47f0-b264-ee44815a417b jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -206,11 +228,11 @@ templates: {% if shuffled_order == 0 %} - A + 1 {% else %} - B + 2 {% endif %}' metadata: !TemplateMetadata @@ -218,14 +240,14 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: random_choice_1_2_2 reference: '' - ece76692-95b0-41a4-acc5-ebb031f8dd35: !Template + d919cd6a-6031-4835-a7a5-f61602cda175: !Template answer_choices: null - id: ece76692-95b0-41a4-acc5-ebb031f8dd35 + id: d919cd6a-6031-4835-a7a5-f61602cda175 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -257,43 +279,21 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: random_choice_1_2 reference: '' - f3cd7d54-90f3-4303-83dc-90606680df99: !Template - answer_choices: A ||| B - id: f3cd7d54-90f3-4303-83dc-90606680df99 - jinja: '{% set shuffled_order = [0, 1] | random %} - - Read the following two sentences. - - {% if shuffled_order == 0 %} - - Sentence A: {{ sentence_good }} - - Sentence B: {{ sentence_bad }} - - {% else %} - - Sentence A: {{ sentence_bad }} - - Sentence B: {{ sentence_good }} - - {% endif %} - - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} + e2ce39cf-1e36-4a2f-b76e-ba40473d4109: !Template + answer_choices: Yes ||| No + id: e2ce39cf-1e36-4a2f-b76e-ba40473d4109 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - {% endif %}' + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: A/B choice randomized (choice order reversed) + original_task: false + name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' reference: '' diff --git a/promptsource/templates/blimp/wh_vs_that_with_gap_long_distance/templates.yaml b/promptsource/templates/blimp/wh_vs_that_with_gap_long_distance/templates.yaml index b6aaec34b..b5e0c7b92 100644 --- a/promptsource/templates/blimp/wh_vs_that_with_gap_long_distance/templates.yaml +++ b/promptsource/templates/blimp/wh_vs_that_with_gap_long_distance/templates.yaml @@ -1,9 +1,9 @@ dataset: blimp subset: wh_vs_that_with_gap_long_distance templates: - 2914cfe5-91ea-40ec-87a7-b812d80ae297: !Template + 03f877b2-092b-49f2-bad1-80b7a136e720: !Template answer_choices: A ||| B - id: 2914cfe5-91ea-40ec-87a7-b812d80ae297 + id: 03f877b2-092b-49f2-bad1-80b7a136e720 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -22,7 +22,7 @@ templates: {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order + Which one is a better sentence of English, A or B? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -37,11 +37,11 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized (choice order reversed) + name: 'A/B choice randomized (choice order: A-B)' reference: '' - 30f395ce-ebb1-4028-9006-1662a1958ca9: !Template + 044612a2-d57b-4534-9eb1-3f6fdf902294: !Template answer_choices: null - id: 30f395ce-ebb1-4028-9006-1662a1958ca9 + id: 044612a2-d57b-4534-9eb1-3f6fdf902294 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -78,9 +78,9 @@ templates: original_task: true name: random_choice_1_2_2 reference: '' - 3c00e58c-042a-4710-b3db-f94b2d61828b: !Template + 19780724-c516-4229-bbc4-53f6055e1076: !Template answer_choices: null - id: 3c00e58c-042a-4710-b3db-f94b2d61828b + id: 19780724-c516-4229-bbc4-53f6055e1076 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer A or B. @@ -117,24 +117,24 @@ templates: original_task: true name: random_choice_A_B reference: '' - 432b04b6-abaf-4d98-afeb-e4deb674c8f6: !Template + 2c516153-a176-4a03-a109-1c7d5d508872: !Template answer_choices: null - id: 432b04b6-abaf-4d98-afeb-e4deb674c8f6 + id: 2c516153-a176-4a03-a109-1c7d5d508872 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -142,11 +142,11 @@ templates: {% if shuffled_order == 0 %} - A + 1 {% else %} - B + 2 {% endif %}' metadata: !TemplateMetadata @@ -154,11 +154,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: random_choice_1_2 reference: '' - 46e50db4-324a-4dca-ae68-af8f4bf7156d: !Template + 358d3f20-8d63-402c-9cf7-d28df1aa4e77: !Template answer_choices: A ||| B - id: 46e50db4-324a-4dca-ae68-af8f4bf7156d + id: 358d3f20-8d63-402c-9cf7-d28df1aa4e77 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -177,7 +177,7 @@ templates: {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order + Which one is a better sentence of English, B or A? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -192,15 +192,15 @@ templates: metrics: - Accuracy original_task: true - name: A/B choice randomized + name: 'A/B choice randomized (choice order: B-A)' reference: '' - 5feecac8-705d-447c-a2f8-8f0986a73f60: !Template + 3de137a6-a2d5-4bb5-b31a-7e2cc7a08828: !Template answer_choices: Yes ||| No - id: 5feecac8-705d-447c-a2f8-8f0986a73f60 + id: 3de137a6-a2d5-4bb5-b31a-7e2cc7a08828 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata @@ -208,11 +208,11 @@ templates: metrics: - Accuracy original_task: false - name: Yes/No binary judgment for bad sentences + name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' reference: '' - 84a826ff-a341-40a5-978a-db148ddaef96: !Template + e73651a0-d884-45b3-8ba4-c26a2c2c2051: !Template answer_choices: Yes ||| No - id: 84a826ff-a341-40a5-978a-db148ddaef96 + id: e73651a0-d884-45b3-8ba4-c26a2c2c2051 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -224,26 +224,58 @@ templates: metrics: - Accuracy original_task: false - name: Yes/No binary judgment for good sentences (choice order reversed) + name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' + reference: '' + f1764f9f-d8c5-49f0-8100-9b344edefef6: !Template + answer_choices: Yes ||| No + id: f1764f9f-d8c5-49f0-8100-9b344edefef6 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' + reference: '' + f762d6f8-3685-4614-9ac2-7b2eeae93415: !Template + answer_choices: Yes ||| No + id: f762d6f8-3685-4614-9ac2-7b2eeae93415 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' reference: '' - 8cdd2963-9f51-4f29-891d-36193ddf7617: !Template + fa4b90ca-133a-4712-bc83-c3280b5518a1: !Template answer_choices: null - id: 8cdd2963-9f51-4f29-891d-36193ddf7617 + id: fa4b90ca-133a-4712-bc83-c3280b5518a1 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -251,11 +283,11 @@ templates: {% if shuffled_order == 0 %} - 1 + A {% else %} - 2 + B {% endif %}' metadata: !TemplateMetadata @@ -263,37 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 - reference: '' - e3311cac-ad90-4a9a-b9b9-24530f9af0c9: !Template - answer_choices: Yes ||| No - id: e3311cac-ad90-4a9a-b9b9-24530f9af0c9 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for bad sentences (choice order reversed) - reference: '' - f8a750a3-a7cc-479a-b4bc-6f14959c9225: !Template - answer_choices: Yes ||| No - id: f8a750a3-a7cc-479a-b4bc-6f14959c9225 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: Yes/No binary judgment for good sentences + name: random_choice_A_B_2 reference: '' From 52d4d1b8d277f566ac1d1089d5e79550b5680300 Mon Sep 17 00:00:00 2001 From: najoungkim Date: Tue, 26 Apr 2022 17:40:36 -0400 Subject: [PATCH 12/16] Fix answer_choices for some templates + better template names --- .../blimp/adjunct_island/templates.yaml | 44 +-- .../anaphor_gender_agreement/templates.yaml | 188 ++++++------- .../anaphor_number_agreement/templates.yaml | 234 ++++++++-------- .../animate_subject_passive/templates.yaml | 238 ++++++++-------- .../animate_subject_trans/templates.yaml | 190 ++++++------- .../templates/blimp/causative/templates.yaml | 206 +++++++------- .../blimp/complex_NP_island/templates.yaml | 222 +++++++-------- .../templates.yaml | 202 +++++++------- .../templates.yaml | 230 ++++++++-------- .../templates.yaml | 186 ++++++------- .../templates.yaml | 170 ++++++------ .../templates.yaml | 234 ++++++++-------- .../templates.yaml | 218 +++++++-------- .../templates.yaml | 204 +++++++------- .../templates.yaml | 218 +++++++-------- .../templates.yaml | 230 ++++++++-------- .../templates.yaml | 174 ++++++------ .../templates.yaml | 224 +++++++-------- .../templates.yaml | 244 ++++++++-------- .../blimp/drop_argument/templates.yaml | 146 +++++----- .../blimp/ellipsis_n_bar_1/templates.yaml | 210 +++++++------- .../blimp/ellipsis_n_bar_2/templates.yaml | 224 +++++++-------- .../templates.yaml | 178 ++++++------ .../templates.yaml | 166 +++++------ .../templates.yaml | 228 +++++++-------- .../templates.yaml | 230 ++++++++-------- .../templates.yaml | 242 ++++++++-------- .../templates/blimp/inchoative/templates.yaml | 214 +++++++------- .../blimp/intransitive/templates.yaml | 212 +++++++------- .../templates.yaml | 176 ++++++------ .../templates.yaml | 208 +++++++------- .../templates.yaml | 198 ++++++------- .../templates.yaml | 190 ++++++------- .../templates.yaml | 206 +++++++------- .../templates.yaml | 216 +++++++-------- .../templates.yaml | 214 +++++++------- .../blimp/npi_present_1/templates.yaml | 218 +++++++-------- .../blimp/npi_present_2/templates.yaml | 186 ++++++------- .../only_npi_licensor_present/templates.yaml | 230 ++++++++-------- .../blimp/only_npi_scope/templates.yaml | 210 +++++++------- .../templates/blimp/passive_1/templates.yaml | 206 +++++++------- .../templates/blimp/passive_2/templates.yaml | 192 ++++++------- .../principle_A_c_command/templates.yaml | 210 +++++++------- .../blimp/principle_A_case_1/templates.yaml | 198 ++++++------- .../blimp/principle_A_case_2/templates.yaml | 200 +++++++------- .../blimp/principle_A_domain_1/templates.yaml | 232 ++++++++-------- .../blimp/principle_A_domain_2/templates.yaml | 208 +++++++------- .../blimp/principle_A_domain_3/templates.yaml | 202 +++++++------- .../principle_A_reconstruction/templates.yaml | 216 +++++++-------- .../templates.yaml | 260 +++++++++--------- .../templates.yaml | 204 +++++++------- .../templates.yaml | 214 +++++++------- .../templates.yaml | 188 ++++++------- .../sentential_subject_island/templates.yaml | 204 +++++++------- .../superlative_quantifiers_1/templates.yaml | 216 +++++++-------- .../superlative_quantifiers_2/templates.yaml | 224 +++++++-------- .../blimp/tough_vs_raising_1/templates.yaml | 172 ++++++------ .../blimp/tough_vs_raising_2/templates.yaml | 192 ++++++------- .../templates/blimp/transitive/templates.yaml | 214 +++++++------- .../templates/blimp/wh_island/templates.yaml | 188 ++++++------- .../wh_questions_object_gap/templates.yaml | 200 +++++++------- .../wh_questions_subject_gap/templates.yaml | 222 +++++++-------- .../templates.yaml | 198 ++++++------- .../blimp/wh_vs_that_no_gap/templates.yaml | 202 +++++++------- .../templates.yaml | 180 ++++++------ .../blimp/wh_vs_that_with_gap/templates.yaml | 216 +++++++-------- .../templates.yaml | 210 +++++++------- 67 files changed, 6863 insertions(+), 6863 deletions(-) diff --git a/promptsource/templates/blimp/adjunct_island/templates.yaml b/promptsource/templates/blimp/adjunct_island/templates.yaml index ead81d2e5..166acc487 100644 --- a/promptsource/templates/blimp/adjunct_island/templates.yaml +++ b/promptsource/templates/blimp/adjunct_island/templates.yaml @@ -37,10 +37,10 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: B-A)' + name: better_B_A reference: '' 6e71cd4b-882c-4458-9cd4-fd83f849ad09: !Template - answer_choices: null + answer_choices: 1 ||| 2 id: 6e71cd4b-882c-4458-9cd4-fd83f849ad09 jinja: '{% set shuffled_order = [0, 1] | random %} @@ -64,11 +64,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -76,10 +76,10 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: grammatical_between_1_2 reference: '' 7ed733fd-524c-4f00-800f-77a7a862ab4a: !Template - answer_choices: null + answer_choices: 1 ||| 2 id: 7ed733fd-524c-4f00-800f-77a7a862ab4a jinja: '{% set shuffled_order = [0, 1] | random %} @@ -103,11 +103,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -115,7 +115,7 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: grammatical_which_one_1_2 reference: '' 8bb3b001-cda4-43a3-8692-f191393bb2ef: !Template answer_choices: Yes ||| No @@ -131,7 +131,7 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' + name: single_sentence_bad_no_yes reference: '' 8f354d74-5ffb-4cbc-b119-b79f2db00df3: !Template answer_choices: Yes ||| No @@ -147,10 +147,10 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' + name: single_sentence_bad_yes_no reference: '' 967ca4e2-162f-4365-a1f5-2c87c0e874ce: !Template - answer_choices: null + answer_choices: A ||| B id: 967ca4e2-162f-4365-a1f5-2c87c0e874ce jinja: '{% set shuffled_order = [0, 1] | random %} @@ -174,11 +174,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -186,7 +186,7 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: grammatical_between_A_B reference: '' a1997343-0a20-4f8c-8939-a512f0f68485: !Template answer_choices: Yes ||| No @@ -202,7 +202,7 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' + name: single_sentence_good_yes_no reference: '' c45d5bac-968e-4e39-96b7-d18a5ddb225c: !Template answer_choices: A ||| B @@ -240,10 +240,10 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: A-B)' + name: better_A_B reference: '' d11a37d7-7c42-4ee0-96c5-ffd24ad7073e: !Template - answer_choices: null + answer_choices: A ||| B id: d11a37d7-7c42-4ee0-96c5-ffd24ad7073e jinja: '{% set shuffled_order = [0, 1] | random %} @@ -267,11 +267,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -279,7 +279,7 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: grammatical_which_one_A_B reference: '' f4f94eda-a7ac-485a-82d1-e36012771762: !Template answer_choices: Yes ||| No @@ -295,5 +295,5 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' + name: single_sentence_good_no_yes reference: '' diff --git a/promptsource/templates/blimp/anaphor_gender_agreement/templates.yaml b/promptsource/templates/blimp/anaphor_gender_agreement/templates.yaml index f056324fc..0cd1085fa 100644 --- a/promptsource/templates/blimp/anaphor_gender_agreement/templates.yaml +++ b/promptsource/templates/blimp/anaphor_gender_agreement/templates.yaml @@ -1,24 +1,24 @@ dataset: blimp subset: anaphor_gender_agreement templates: - 0c692382-5675-48f9-81a7-a9ef7cc18c43: !Template - answer_choices: null - id: 0c692382-5675-48f9-81a7-a9ef7cc18c43 + 2ce6d65f-1230-47ee-8900-685880bce0c3: !Template + answer_choices: 1 ||| 2 + id: 2ce6d65f-1230-47ee-8900-685880bce0c3 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -26,11 +26,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -38,11 +38,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: grammatical_between_1_2 reference: '' - 162a2f7a-afaf-4af9-9a2c-c6d6dd32a89e: !Template + 2e1d6529-58d2-454d-a1fe-49ddec83b3fd: !Template answer_choices: A ||| B - id: 162a2f7a-afaf-4af9-9a2c-c6d6dd32a89e + id: 2e1d6529-58d2-454d-a1fe-49ddec83b3fd jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -76,11 +76,27 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: A-B)' + name: better_A_B reference: '' - 2ae00b0c-a3b8-4adf-a828-08bdc6531814: !Template + 36a58157-aa61-421d-ba18-053a2c3f3456: !Template answer_choices: Yes ||| No - id: 2ae00b0c-a3b8-4adf-a828-08bdc6531814 + id: 36a58157-aa61-421d-ba18-053a2c3f3456 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 4e72a002-ae42-4a95-b390-b008922ebd45: !Template + answer_choices: Yes ||| No + id: 4e72a002-ae42-4a95-b390-b008922ebd45 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -92,26 +108,26 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' + name: single_sentence_bad_no_yes reference: '' - 2d83a989-9bf8-48fb-909a-0ac5aa334a51: !Template - answer_choices: null - id: 2d83a989-9bf8-48fb-909a-0ac5aa334a51 + 5380667f-a977-42ac-b00b-a69ee9f8f8fd: !Template + answer_choices: A ||| B + id: 5380667f-a977-42ac-b00b-a69ee9f8f8fd jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -119,11 +135,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -131,43 +147,66 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: grammatical_between_A_B reference: '' - 9aae2f9d-51f8-4388-a3b6-3014bd9ee634: !Template + 57368d1e-d9e9-4e23-b6ca-a73f816c2036: !Template answer_choices: Yes ||| No - id: 9aae2f9d-51f8-4388-a3b6-3014bd9ee634 + id: 57368d1e-d9e9-4e23-b6ca-a73f816c2036 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' + name: single_sentence_good_no_yes reference: '' - bdfe7076-7af9-454d-8984-3b8a8f5f96e9: !Template - answer_choices: Yes ||| No - id: bdfe7076-7af9-454d-8984-3b8a8f5f96e9 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + 5fab1923-c176-4e58-a2e0-238141d0b209: !Template + answer_choices: 1 ||| 2 + id: 5fab1923-c176-4e58-a2e0-238141d0b209 + jinja: '{% set shuffled_order = [0, 1] | random %} - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' + original_task: true + name: grammatical_which_one_1_2 reference: '' - cfbafd1a-03b6-41bd-a8cc-2ba4ccc480c3: !Template + 6157d8d7-7889-4c4e-be23-2ce3c626ee4d: !Template answer_choices: A ||| B - id: cfbafd1a-03b6-41bd-a8cc-2ba4ccc480c3 + id: 6157d8d7-7889-4c4e-be23-2ce3c626ee4d jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -201,11 +240,11 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: B-A)' + name: better_B_A reference: '' - d2945120-7955-4217-8d8f-3c8f824b307e: !Template + 76ba2300-517d-406a-b07b-f0ff0eee62e4: !Template answer_choices: Yes ||| No - id: d2945120-7955-4217-8d8f-3c8f824b307e + id: 76ba2300-517d-406a-b07b-f0ff0eee62e4 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -217,11 +256,11 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' + name: single_sentence_good_yes_no reference: '' - e1ede558-4086-4430-8cea-3b0b86c1e8b2: !Template - answer_choices: null - id: e1ede558-4086-4430-8cea-3b0b86c1e8b2 + be4d8987-d4fb-4562-b99d-6625e2ab5f76: !Template + answer_choices: A ||| B + id: be4d8987-d4fb-4562-b99d-6625e2ab5f76 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer A or B. @@ -244,50 +283,11 @@ templates: {% if shuffled_order == 0 %} - A - - {% else %} - - B - - {% endif %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: random_choice_A_B - reference: '' - ebe1dc30-087d-45ba-941b-8d7b43083e0d: !Template - answer_choices: null - id: ebe1dc30-087d-45ba-941b-8d7b43083e0d - jinja: '{% set shuffled_order = [0, 1] | random %} - - Between the two sentences, which one is grammatical? Please answer 1 or 2. - - {% if shuffled_order == 0 %} - - 1: {{ sentence_good }} - - 2: {{ sentence_bad }} - - {% else %} - - 1: {{ sentence_bad }} - - 2: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -295,5 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: grammatical_which_one_A_B reference: '' diff --git a/promptsource/templates/blimp/anaphor_number_agreement/templates.yaml b/promptsource/templates/blimp/anaphor_number_agreement/templates.yaml index db985435c..f7e2393d5 100644 --- a/promptsource/templates/blimp/anaphor_number_agreement/templates.yaml +++ b/promptsource/templates/blimp/anaphor_number_agreement/templates.yaml @@ -1,25 +1,48 @@ dataset: blimp subset: anaphor_number_agreement templates: - 0b145f73-63d8-4d19-8aa7-7f69909ac2e1: !Template - answer_choices: Yes ||| No - id: 0b145f73-63d8-4d19-8aa7-7f69909ac2e1 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + 091d3d5c-a07f-4467-ba2f-d6b1923b22a1: !Template + answer_choices: 1 ||| 2 + id: 091d3d5c-a07f-4467-ba2f-d6b1923b22a1 + jinja: '{% set shuffled_order = [0, 1] | random %} - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' + original_task: true + name: grammatical_which_one_1_2 reference: '' - 1033ad24-1228-462a-97bb-e84dd3f484c1: !Template + 128cee90-722e-4dde-91d2-3b4dade54971: !Template answer_choices: A ||| B - id: 1033ad24-1228-462a-97bb-e84dd3f484c1 + id: 128cee90-722e-4dde-91d2-3b4dade54971 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -53,54 +76,53 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: A-B)' + name: better_A_B reference: '' - 165fdaa5-19fc-49d3-84ab-657c1f108dc8: !Template + 19479faa-c653-456d-8267-f835593aa75f: !Template answer_choices: Yes ||| No - id: 165fdaa5-19fc-49d3-84ab-657c1f108dc8 + id: 19479faa-c653-456d-8267-f835593aa75f jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' + name: single_sentence_bad_yes_no reference: '' - 1d973777-1811-4c8b-8854-e6473263e524: !Template - answer_choices: null - id: 1d973777-1811-4c8b-8854-e6473263e524 + 32bd7408-29f8-4a8e-ab14-940ac50cdae8: !Template + answer_choices: A ||| B + id: 32bd7408-29f8-4a8e-ab14-940ac50cdae8 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -108,26 +130,26 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: better_B_A reference: '' - 4aa5cd48-6914-43bb-917f-962b4cf1303a: !Template - answer_choices: null - id: 4aa5cd48-6914-43bb-917f-962b4cf1303a + 6dadf8d0-4cff-4aa8-b54a-c4ed82f23cb2: !Template + answer_choices: A ||| B + id: 6dadf8d0-4cff-4aa8-b54a-c4ed82f23cb2 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -135,11 +157,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -147,42 +169,26 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 - reference: '' - 576c4938-8ff8-40d9-b7fc-b44c866ba1d1: !Template - answer_choices: Yes ||| No - id: 576c4938-8ff8-40d9-b7fc-b44c866ba1d1 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' + name: grammatical_which_one_A_B reference: '' - 8c566a9d-fbe5-4087-b672-b83116ca017c: !Template - answer_choices: null - id: 8c566a9d-fbe5-4087-b672-b83116ca017c + 7033308d-63b2-4f28-b0fa-2aec27de9e25: !Template + answer_choices: 1 ||| 2 + id: 7033308d-63b2-4f28-b0fa-2aec27de9e25 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -190,11 +196,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -202,47 +208,48 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: grammatical_between_1_2 reference: '' - 9fc3ec60-d67f-47af-ab3a-d19f2ad9f78b: !Template + 92dc5280-6832-4e8a-85bd-c5beb4bd5824: !Template answer_choices: Yes ||| No - id: 9fc3ec60-d67f-47af-ab3a-d19f2ad9f78b + id: 92dc5280-6832-4e8a-85bd-c5beb4bd5824 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' + name: single_sentence_good_yes_no reference: '' - cca53f6a-1f7c-4b2a-bae6-bab87923ec8d: !Template + 9cd88feb-dc16-40c6-b642-ceb351edda27: !Template answer_choices: A ||| B - id: cca53f6a-1f7c-4b2a-bae6-bab87923ec8d + id: 9cd88feb-dc16-40c6-b642-ceb351edda27 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -256,44 +263,37 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: B-A)' + name: grammatical_between_A_B reference: '' - e379133e-23b1-4ce8-a978-3f3027fea41b: !Template - answer_choices: null - id: e379133e-23b1-4ce8-a978-3f3027fea41b - jinja: '{% set shuffled_order = [0, 1] | random %} - - Between the two sentences, which one is grammatical? Please answer 1 or 2. - - {% if shuffled_order == 0 %} - - 1: {{ sentence_good }} - - 2: {{ sentence_bad }} - - {% else %} - - 1: {{ sentence_bad }} - - 2: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - 1 - - {% else %} + 9f6b4bab-73d8-45ce-8b30-b515c6a501cf: !Template + answer_choices: Yes ||| No + id: 9f6b4bab-73d8-45ce-8b30-b515c6a501cf + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - 2 + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes + reference: '' + cab3fcac-1f71-4f62-b10c-2e8997ada36a: !Template + answer_choices: Yes ||| No + id: cab3fcac-1f71-4f62-b10c-2e8997ada36a + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - {% endif %}' + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: random_choice_1_2_2 + original_task: false + name: single_sentence_bad_no_yes reference: '' diff --git a/promptsource/templates/blimp/animate_subject_passive/templates.yaml b/promptsource/templates/blimp/animate_subject_passive/templates.yaml index 7b2eadcea..a97788a3f 100644 --- a/promptsource/templates/blimp/animate_subject_passive/templates.yaml +++ b/promptsource/templates/blimp/animate_subject_passive/templates.yaml @@ -1,9 +1,48 @@ dataset: blimp subset: animate_subject_passive templates: - 453b7cec-e022-4754-8227-f1b16412f82d: !Template + 1742794e-41e7-4865-a7e4-fc9d3b56a491: !Template + answer_choices: A ||| B + id: 1742794e-41e7-4865-a7e4-fc9d3b56a491 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_A_B + reference: '' + 4fec2ad9-0944-4cc8-9535-f089cf3ef14e: !Template answer_choices: Yes ||| No - id: 453b7cec-e022-4754-8227-f1b16412f82d + id: 4fec2ad9-0944-4cc8-9535-f089cf3ef14e jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -15,11 +54,43 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' + name: single_sentence_bad_yes_no + reference: '' + 59ee660e-a457-4325-813c-abcd6e40d15a: !Template + answer_choices: Yes ||| No + id: 59ee660e-a457-4325-813c-abcd6e40d15a + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 6ed10f7d-da3e-41bb-8b1a-47489f7f5bca: !Template + answer_choices: Yes ||| No + id: 6ed10f7d-da3e-41bb-8b1a-47489f7f5bca + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes reference: '' - 64a41fa3-dbf6-4082-94f0-c23b58df709e: !Template + 75146e21-4abe-4a21-98bf-72bb8c5458f0: !Template answer_choices: A ||| B - id: 64a41fa3-dbf6-4082-94f0-c23b58df709e + id: 75146e21-4abe-4a21-98bf-72bb8c5458f0 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -38,7 +109,7 @@ templates: {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order + Which one is a better sentence of English, B or A? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -53,14 +124,30 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: A-B)' + name: better_B_A + reference: '' + 76f0bb6b-ce88-4c2b-9b7c-37a3d744b65e: !Template + answer_choices: Yes ||| No + id: 76f0bb6b-ce88-4c2b-9b7c-37a3d744b65e + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes reference: '' - 7a84023e-edd0-4b33-88ac-295fed2391e9: !Template - answer_choices: null - id: 7a84023e-edd0-4b33-88ac-295fed2391e9 + 82750934-e6fa-46b5-8d43-2cc53b3bcc9a: !Template + answer_choices: 1 ||| 2 + id: 82750934-e6fa-46b5-8d43-2cc53b3bcc9a jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -80,11 +167,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -92,14 +179,14 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: grammatical_which_one_1_2 reference: '' - 881cfd60-e91a-4314-a235-6196cd6ff251: !Template - answer_choices: null - id: 881cfd60-e91a-4314-a235-6196cd6ff251 + af19a07b-836c-4a61-a195-91a3d3d3eefb: !Template + answer_choices: A ||| B + id: af19a07b-836c-4a61-a195-91a3d3d3eefb jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -119,11 +206,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -131,30 +218,14 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: grammatical_which_one_A_B reference: '' - 97cb7e83-2385-461e-bc05-dc5511aaff5f: !Template - answer_choices: Yes ||| No - id: 97cb7e83-2385-461e-bc05-dc5511aaff5f - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' - reference: '' - 9a2eb37a-8256-4607-90c4-14ba522942b9: !Template - answer_choices: null - id: 9a2eb37a-8256-4607-90c4-14ba522942b9 + d5f377f8-8581-4858-95e5-890368ce7384: !Template + answer_choices: 1 ||| 2 + id: d5f377f8-8581-4858-95e5-890368ce7384 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -174,82 +245,11 @@ templates: {% if shuffled_order == 0 %} - 1 - - {% else %} - - 2 - - {% endif %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: random_choice_1_2 - reference: '' - b89dfc16-3601-4ab7-9275-0be0475d1f57: !Template - answer_choices: Yes ||| No - id: b89dfc16-3601-4ab7-9275-0be0475d1f57 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' - reference: '' - b985ea33-3169-4d14-9727-1c1e1681d33c: !Template - answer_choices: Yes ||| No - id: b985ea33-3169-4d14-9727-1c1e1681d33c - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' - reference: '' - d1b2916a-daee-4424-9238-d7b40ca79aaa: !Template - answer_choices: null - id: d1b2916a-daee-4424-9238-d7b40ca79aaa - jinja: '{% set shuffled_order = [0, 1] | random %} - - Which one of the following sentences is grammatical? Please answer A or B. - - {% if shuffled_order == 0 %} - - A: {{ sentence_good }} - - B: {{ sentence_bad }} - - {% else %} - - A: {{ sentence_bad }} - - B: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -257,11 +257,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: grammatical_between_1_2 reference: '' - ee24cebd-6a6f-4c39-8fa2-ec58545dcc88: !Template + f27c2d94-0531-4717-9a66-6b59d8f06b91: !Template answer_choices: A ||| B - id: ee24cebd-6a6f-4c39-8fa2-ec58545dcc88 + id: f27c2d94-0531-4717-9a66-6b59d8f06b91 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -280,7 +280,7 @@ templates: {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order + Which one is a better sentence of English, A or B? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -295,5 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: B-A)' + name: better_A_B reference: '' diff --git a/promptsource/templates/blimp/animate_subject_trans/templates.yaml b/promptsource/templates/blimp/animate_subject_trans/templates.yaml index c51a25f15..4162644cc 100644 --- a/promptsource/templates/blimp/animate_subject_trans/templates.yaml +++ b/promptsource/templates/blimp/animate_subject_trans/templates.yaml @@ -1,45 +1,30 @@ dataset: blimp subset: animate_subject_trans templates: - 073e4b7b-95bb-4dfe-849d-7b97e58ee49b: !Template - answer_choices: Yes ||| No - id: 073e4b7b-95bb-4dfe-849d-7b97e58ee49b - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' - reference: '' - 304746e1-9763-43c3-b17a-273c149c4dbe: !Template + 23cb5415-e6bb-422f-89d5-65d673b56b66: !Template answer_choices: A ||| B - id: 304746e1-9763-43c3-b17a-273c149c4dbe + id: 23cb5415-e6bb-422f-89d5-65d673b56b66 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -53,26 +38,26 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: A-B)' + name: grammatical_which_one_A_B reference: '' - 30aeeae7-ca4c-400c-9544-2d8ed77a9816: !Template - answer_choices: null - id: 30aeeae7-ca4c-400c-9544-2d8ed77a9816 + 41b54450-d6d8-44fe-8c9f-d9d75583f7e7: !Template + answer_choices: 1 ||| 2 + id: 41b54450-d6d8-44fe-8c9f-d9d75583f7e7 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -80,11 +65,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -92,11 +77,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: grammatical_which_one_1_2 reference: '' - 44e124f6-1877-4f30-ae53-0858dde71199: !Template + 53ed147d-3bbc-4887-8dc2-d146237098ed: !Template answer_choices: Yes ||| No - id: 44e124f6-1877-4f30-ae53-0858dde71199 + id: 53ed147d-3bbc-4887-8dc2-d146237098ed jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -108,38 +93,37 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' + name: single_sentence_bad_yes_no reference: '' - 72835b3d-b9ac-42ea-a611-16e5571244cd: !Template - answer_choices: null - id: 72835b3d-b9ac-42ea-a611-16e5571244cd + 750906cd-1c40-4e75-8715-8a27bacd10ac: !Template + answer_choices: A ||| B + id: 750906cd-1c40-4e75-8715-8a27bacd10ac jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -147,26 +131,26 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: better_B_A reference: '' - 8d86aa5d-10fd-495b-84f3-1c60d55304f8: !Template - answer_choices: null - id: 8d86aa5d-10fd-495b-84f3-1c60d55304f8 + 87ed0b72-98c5-406d-8142-f93dc73268fd: !Template + answer_choices: A ||| B + id: 87ed0b72-98c5-406d-8142-f93dc73268fd jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -174,11 +158,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -186,27 +170,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: grammatical_between_A_B reference: '' - a62914b0-cc0d-47e0-ac13-8f12f99fda7e: !Template - answer_choices: Yes ||| No - id: a62914b0-cc0d-47e0-ac13-8f12f99fda7e - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' - reference: '' - a73020d6-d1bb-45d4-9dec-6c226864ef6a: !Template + 8e35221d-80fb-41c6-a730-18584e445119: !Template answer_choices: A ||| B - id: a73020d6-d1bb-45d4-9dec-6c226864ef6a + id: 8e35221d-80fb-41c6-a730-18584e445119 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -225,7 +193,7 @@ templates: {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order + Which one is a better sentence of English, A or B? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -240,11 +208,43 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: B-A)' + name: better_A_B + reference: '' + 97214438-78c8-45f7-b70c-2d13814d232f: !Template + answer_choices: Yes ||| No + id: 97214438-78c8-45f7-b70c-2d13814d232f + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes reference: '' - c185d3df-50a5-433a-a904-c56b95e5cb29: !Template - answer_choices: null - id: c185d3df-50a5-433a-a904-c56b95e5cb29 + a5a3b920-9d1f-4813-b48f-26304d00725d: !Template + answer_choices: Yes ||| No + id: a5a3b920-9d1f-4813-b48f-26304d00725d + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes + reference: '' + cc199813-7feb-4989-a35b-397ce431524e: !Template + answer_choices: 1 ||| 2 + id: cc199813-7feb-4989-a35b-397ce431524e jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -267,11 +267,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -279,15 +279,15 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: grammatical_between_1_2 reference: '' - dd656c7e-2716-4791-81a8-2e9945a4345d: !Template + ddfc92aa-0952-4bfc-9631-cd0dd81fc85b: !Template answer_choices: Yes ||| No - id: dd656c7e-2716-4791-81a8-2e9945a4345d + id: ddfc92aa-0952-4bfc-9631-cd0dd81fc85b jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -295,5 +295,5 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' + name: single_sentence_good_yes_no reference: '' diff --git a/promptsource/templates/blimp/causative/templates.yaml b/promptsource/templates/blimp/causative/templates.yaml index fa2f0fab1..000728461 100644 --- a/promptsource/templates/blimp/causative/templates.yaml +++ b/promptsource/templates/blimp/causative/templates.yaml @@ -1,48 +1,25 @@ dataset: blimp subset: causative templates: - 0bc2dedf-4691-4b25-a06a-ecb619c36087: !Template - answer_choices: null - id: 0bc2dedf-4691-4b25-a06a-ecb619c36087 - jinja: '{% set shuffled_order = [0, 1] | random %} - - Which one of the following sentences is grammatical? Please answer 1 or 2. - - {% if shuffled_order == 0 %} - - 1: {{ sentence_good }} - - 2: {{ sentence_bad }} - - {% else %} - - 1: {{ sentence_bad }} - - 2: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - 1 - - {% else %} - - 2 + 15261315-cf12-4730-9cd1-29b065c485d8: !Template + answer_choices: Yes ||| No + id: 15261315-cf12-4730-9cd1-29b065c485d8 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - {% endif %}' + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: random_choice_1_2 + original_task: false + name: single_sentence_bad_no_yes reference: '' - 3ede30de-975e-44e2-94e4-106b191d4bf4: !Template - answer_choices: null - id: 3ede30de-975e-44e2-94e4-106b191d4bf4 + 28913271-0dfa-4172-a2b2-81cc02a55740: !Template + answer_choices: A ||| B + id: 28913271-0dfa-4172-a2b2-81cc02a55740 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer A or B. @@ -65,11 +42,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -77,27 +54,27 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: grammatical_between_A_B reference: '' - 8a92bd39-198d-4233-a286-932d01339249: !Template + 311b5f2e-2595-4c2c-9238-d3a9b92ce09a: !Template answer_choices: Yes ||| No - id: 8a92bd39-198d-4233-a286-932d01339249 + id: 311b5f2e-2595-4c2c-9238-d3a9b92ce09a jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' + name: single_sentence_bad_yes_no reference: '' - 9585c165-4da8-4f13-8952-cb7dd8249347: !Template + 42efe756-1e77-4d04-8fbc-4a805cd25041: !Template answer_choices: A ||| B - id: 9585c165-4da8-4f13-8952-cb7dd8249347 + id: 42efe756-1e77-4d04-8fbc-4a805cd25041 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -131,11 +108,27 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: B-A)' + name: better_B_A reference: '' - 9f141a7b-43ec-44c7-8d46-10b4163ae970: !Template + 4c5cf40e-e28c-47bb-a211-66ad8a2c5a25: !Template + answer_choices: Yes ||| No + id: 4c5cf40e-e28c-47bb-a211-66ad8a2c5a25 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 5f94bead-d030-4bd3-b3ad-c609cfa1ecb6: !Template answer_choices: A ||| B - id: 9f141a7b-43ec-44c7-8d46-10b4163ae970 + id: 5f94bead-d030-4bd3-b3ad-c609cfa1ecb6 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -169,58 +162,81 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: A-B)' + name: better_A_B reference: '' - a99bfe8d-3ed3-46bf-8008-87f41cab19b8: !Template - answer_choices: Yes ||| No - id: a99bfe8d-3ed3-46bf-8008-87f41cab19b8 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + b95c004b-a7bb-4abe-99b6-f1ceeff1997c: !Template + answer_choices: 1 ||| 2 + id: b95c004b-a7bb-4abe-99b6-f1ceeff1997c + jinja: '{% set shuffled_order = [0, 1] | random %} - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' + original_task: true + name: grammatical_between_1_2 reference: '' - be157f6c-df7a-4ccf-887a-595ea99d032d: !Template + cc192df1-d684-49ac-a4cc-ac53db84ae73: !Template answer_choices: Yes ||| No - id: be157f6c-df7a-4ccf-887a-595ea99d032d + id: cc192df1-d684-49ac-a4cc-ac53db84ae73 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' + name: single_sentence_good_no_yes reference: '' - d62aa60b-e51d-4353-9246-eedbfd0526dc: !Template - answer_choices: null - id: d62aa60b-e51d-4353-9246-eedbfd0526dc + d19f516c-a99b-4388-8b5d-84731f619cee: !Template + answer_choices: A ||| B + id: d19f516c-a99b-4388-8b5d-84731f619cee jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -228,11 +244,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -240,26 +256,26 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: grammatical_which_one_A_B reference: '' - ed22a914-badf-4ced-a8b9-8c02c850b535: !Template - answer_choices: null - id: ed22a914-badf-4ced-a8b9-8c02c850b535 + db00362c-7411-476d-9fe7-38a647a9a7e8: !Template + answer_choices: 1 ||| 2 + id: db00362c-7411-476d-9fe7-38a647a9a7e8 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -267,11 +283,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -279,21 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B - reference: '' - f362fc21-588d-40bc-a93d-e4f9eaa1bea6: !Template - answer_choices: Yes ||| No - id: f362fc21-588d-40bc-a93d-e4f9eaa1bea6 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' + name: grammatical_which_one_1_2 reference: '' diff --git a/promptsource/templates/blimp/complex_NP_island/templates.yaml b/promptsource/templates/blimp/complex_NP_island/templates.yaml index 39a680bbb..aa14a9b30 100644 --- a/promptsource/templates/blimp/complex_NP_island/templates.yaml +++ b/promptsource/templates/blimp/complex_NP_island/templates.yaml @@ -1,64 +1,9 @@ dataset: blimp subset: complex_NP_island templates: - 041bbb38-1ec7-4ebd-b2dd-89a3af095309: !Template - answer_choices: Yes ||| No - id: 041bbb38-1ec7-4ebd-b2dd-89a3af095309 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' - reference: '' - 0494420d-2b79-48b3-b754-764c17dc5444: !Template - answer_choices: null - id: 0494420d-2b79-48b3-b754-764c17dc5444 - jinja: '{% set shuffled_order = [0, 1] | random %} - - Which one of the following sentences is grammatical? Please answer A or B. - - {% if shuffled_order == 0 %} - - A: {{ sentence_good }} - - B: {{ sentence_bad }} - - {% else %} - - A: {{ sentence_bad }} - - B: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - A - - {% else %} - - B - - {% endif %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: random_choice_A_B - reference: '' - 22df7965-9604-4201-a824-daeb432efe65: !Template + 0495b672-0eeb-4486-8121-935a3bdb4a2d: !Template answer_choices: A ||| B - id: 22df7965-9604-4201-a824-daeb432efe65 + id: 0495b672-0eeb-4486-8121-935a3bdb4a2d jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -77,7 +22,7 @@ templates: {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order + Which one is a better sentence of English, A or B? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -92,26 +37,26 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: B-A)' + name: better_A_B reference: '' - 344d25cb-ed7c-4884-8c80-157dff563b2a: !Template - answer_choices: null - id: 344d25cb-ed7c-4884-8c80-157dff563b2a + 0dbb4a55-45cd-46b7-8a0a-78580888d312: !Template + answer_choices: A ||| B + id: 0dbb4a55-45cd-46b7-8a0a-78580888d312 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -119,11 +64,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -131,11 +76,27 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: grammatical_between_A_B + reference: '' + 0fb945e8-2f72-4622-a364-0716fe4c6a17: !Template + answer_choices: Yes ||| No + id: 0fb945e8-2f72-4622-a364-0716fe4c6a17 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes reference: '' - 434e2faa-3d3e-4a3e-84fe-bdfee866b08a: !Template - answer_choices: null - id: 434e2faa-3d3e-4a3e-84fe-bdfee866b08a + 296e0a9a-9eff-4ee9-9740-a22065d2b097: !Template + answer_choices: 1 ||| 2 + id: 296e0a9a-9eff-4ee9-9740-a22065d2b097 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -158,11 +119,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -170,11 +131,27 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: grammatical_between_1_2 reference: '' - 6089eaff-0e6d-4507-a055-1b271d51d37b: !Template + 730ab874-f5d2-4527-9a7b-41f9474f6de1: !Template + answer_choices: Yes ||| No + id: 730ab874-f5d2-4527-9a7b-41f9474f6de1 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 7beba882-e2e7-4f13-9599-f156d83b819c: !Template answer_choices: A ||| B - id: 6089eaff-0e6d-4507-a055-1b271d51d37b + id: 7beba882-e2e7-4f13-9599-f156d83b819c jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -193,7 +170,7 @@ templates: {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order + Which one is a better sentence of English, B or A? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -208,30 +185,14 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: A-B)' + name: better_B_A reference: '' - 7cd043fd-7026-428c-9b18-5416682430d0: !Template - answer_choices: Yes ||| No - id: 7cd043fd-7026-428c-9b18-5416682430d0 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' - reference: '' - 96afc9ce-f867-41b7-a52f-5e67f280b22c: !Template - answer_choices: null - id: 96afc9ce-f867-41b7-a52f-5e67f280b22c + 8798f8e4-7b90-435e-9630-fbfdb187ee46: !Template + answer_choices: A ||| B + id: 8798f8e4-7b90-435e-9630-fbfdb187ee46 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -251,11 +212,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -263,27 +224,27 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: grammatical_which_one_A_B reference: '' - c7644486-bcf6-432c-98e8-a5853871ae56: !Template + b77195ba-a7e0-4225-a343-6f0239621f04: !Template answer_choices: Yes ||| No - id: c7644486-bcf6-432c-98e8-a5853871ae56 + id: b77195ba-a7e0-4225-a343-6f0239621f04 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' + name: single_sentence_bad_yes_no reference: '' - e037553c-43a2-4ab4-b432-e35bcc112630: !Template + f09f244c-048a-4591-aee2-bff934b44cdd: !Template answer_choices: Yes ||| No - id: e037553c-43a2-4ab4-b432-e35bcc112630 + id: f09f244c-048a-4591-aee2-bff934b44cdd jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -295,5 +256,44 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' + name: single_sentence_good_no_yes + reference: '' + f26a0feb-2759-4166-bb10-67337ca716a0: !Template + answer_choices: 1 ||| 2 + id: f26a0feb-2759-4166-bb10-67337ca716a0 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_1_2 reference: '' diff --git a/promptsource/templates/blimp/coordinate_structure_constraint_complex_left_branch/templates.yaml b/promptsource/templates/blimp/coordinate_structure_constraint_complex_left_branch/templates.yaml index 734b94e3a..458a9aeee 100644 --- a/promptsource/templates/blimp/coordinate_structure_constraint_complex_left_branch/templates.yaml +++ b/promptsource/templates/blimp/coordinate_structure_constraint_complex_left_branch/templates.yaml @@ -1,9 +1,25 @@ dataset: blimp subset: coordinate_structure_constraint_complex_left_branch templates: - 21514070-d6e2-48ec-b43f-3447b8ffd989: !Template - answer_choices: null - id: 21514070-d6e2-48ec-b43f-3447b8ffd989 + 11622e6e-0c8c-49e7-b443-df3eb22aba58: !Template + answer_choices: Yes ||| No + id: 11622e6e-0c8c-49e7-b443-df3eb22aba58 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes + reference: '' + 30800a25-611a-48e3-b7cd-61484055d1da: !Template + answer_choices: 1 ||| 2 + id: 30800a25-611a-48e3-b7cd-61484055d1da jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -26,11 +42,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -38,30 +54,14 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 - reference: '' - 25208006-931f-47cd-8912-1f2aa6b25875: !Template - answer_choices: Yes ||| No - id: 25208006-931f-47cd-8912-1f2aa6b25875 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' + name: grammatical_between_1_2 reference: '' - 25cb7f00-b9f8-4a7c-94da-3d4ca96d2fe5: !Template - answer_choices: null - id: 25cb7f00-b9f8-4a7c-94da-3d4ca96d2fe5 + 4d99c065-4b45-4ca2-9899-6630ec7c085d: !Template + answer_choices: A ||| B + id: 4d99c065-4b45-4ca2-9899-6630ec7c085d jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -81,11 +81,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -93,31 +93,48 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: grammatical_between_A_B reference: '' - 2e720290-257e-4105-bcff-6ae4307f0d4f: !Template - answer_choices: A ||| B - id: 2e720290-257e-4105-bcff-6ae4307f0d4f + 73be937e-8c25-44f5-8acc-6db8329c7261: !Template + answer_choices: Yes ||| No + id: 73be937e-8c25-44f5-8acc-6db8329c7261 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 7569112f-12f1-4e92-b01c-f24c9e2bf7f8: !Template + answer_choices: 1 ||| 2 + id: 7569112f-12f1-4e92-b01c-f24c9e2bf7f8 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -131,38 +148,53 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: B-A)' + name: grammatical_which_one_1_2 + reference: '' + 84f173bc-8484-4070-9f38-ca3e837c4701: !Template + answer_choices: Yes ||| No + id: 84f173bc-8484-4070-9f38-ca3e837c4701 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes reference: '' - 30e006b0-14e1-49cf-8a95-6eded8256a86: !Template - answer_choices: null - id: 30e006b0-14e1-49cf-8a95-6eded8256a86 + b707cddf-b388-4b00-80ec-b99b401e00c3: !Template + answer_choices: A ||| B + id: b707cddf-b388-4b00-80ec-b99b401e00c3 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -170,11 +202,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: better_A_B reference: '' - 860e79dd-3638-47fb-9959-595022d60c68: !Template + b86b71d6-2b94-4129-8b6a-254b4c1fe446: !Template answer_choices: A ||| B - id: 860e79dd-3638-47fb-9959-595022d60c68 + id: b86b71d6-2b94-4129-8b6a-254b4c1fe446 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -193,7 +225,7 @@ templates: {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order + Which one is a better sentence of English, B or A? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -208,46 +240,14 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: A-B)' - reference: '' - 96bedf82-1010-432e-8aa1-566ebebd6f73: !Template - answer_choices: Yes ||| No - id: 96bedf82-1010-432e-8aa1-566ebebd6f73 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' - reference: '' - b08e28a2-6e94-4882-8d17-6e89c1f36cf1: !Template - answer_choices: Yes ||| No - id: b08e28a2-6e94-4882-8d17-6e89c1f36cf1 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' + name: better_B_A reference: '' - c4812c70-7a21-4dab-8136-047ed2570fee: !Template - answer_choices: null - id: c4812c70-7a21-4dab-8136-047ed2570fee + cc916b85-8337-446a-87df-64a2b3917783: !Template + answer_choices: A ||| B + id: cc916b85-8337-446a-87df-64a2b3917783 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -267,11 +267,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -279,15 +279,15 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: grammatical_which_one_A_B reference: '' - e11601b3-da0d-4e1f-a54e-9603a2ba6392: !Template + db2bcadb-fcfe-4f86-946f-aa7d1f852d2e: !Template answer_choices: Yes ||| No - id: e11601b3-da0d-4e1f-a54e-9603a2ba6392 + id: db2bcadb-fcfe-4f86-946f-aa7d1f852d2e jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata @@ -295,5 +295,5 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' + name: single_sentence_bad_yes_no reference: '' diff --git a/promptsource/templates/blimp/coordinate_structure_constraint_object_extraction/templates.yaml b/promptsource/templates/blimp/coordinate_structure_constraint_object_extraction/templates.yaml index 695e29b23..fc33800d4 100644 --- a/promptsource/templates/blimp/coordinate_structure_constraint_object_extraction/templates.yaml +++ b/promptsource/templates/blimp/coordinate_structure_constraint_object_extraction/templates.yaml @@ -1,36 +1,51 @@ dataset: blimp subset: coordinate_structure_constraint_object_extraction templates: - 1f3bb98a-03f9-482b-84e7-b73366fd9f92: !Template - answer_choices: null - id: 1f3bb98a-03f9-482b-84e7-b73366fd9f92 + 1c49f77a-fe58-4439-9d94-0f4e1d3b6bef: !Template + answer_choices: Yes ||| No + id: 1c49f77a-fe58-4439-9d94-0f4e1d3b6bef + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 2fd5b072-efa8-4704-a830-d0af484a7bad: !Template + answer_choices: A ||| B + id: 2fd5b072-efa8-4704-a830-d0af484a7bad jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -38,31 +53,32 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: better_B_A reference: '' - 56ffab74-f1b1-4302-9eb4-23b15809583c: !Template + 32b4fcc3-22a2-4721-9c76-f4359f926274: !Template answer_choices: A ||| B - id: 56ffab74-f1b1-4302-9eb4-23b15809583c + id: 32b4fcc3-22a2-4721-9c76-f4359f926274 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -76,11 +92,27 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: B-A)' + name: grammatical_between_A_B + reference: '' + 35813dd3-145c-4d35-800d-1f84f0a5b13d: !Template + answer_choices: Yes ||| No + id: 35813dd3-145c-4d35-800d-1f84f0a5b13d + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no reference: '' - 82522714-9460-4ef1-a7f3-076c3e6e86c5: !Template + 87e44d3c-fff8-4dae-bb46-117f28266aa1: !Template answer_choices: A ||| B - id: 82522714-9460-4ef1-a7f3-076c3e6e86c5 + id: 87e44d3c-fff8-4dae-bb46-117f28266aa1 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -114,11 +146,11 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: A-B)' + name: better_A_B reference: '' - 86530617-f0eb-4aea-b5eb-32ce93b77ff4: !Template - answer_choices: null - id: 86530617-f0eb-4aea-b5eb-32ce93b77ff4 + 8a72ad22-1fb5-4d3d-af85-cfe0d676c8d8: !Template + answer_choices: 1 ||| 2 + id: 8a72ad22-1fb5-4d3d-af85-cfe0d676c8d8 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -141,11 +173,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -153,62 +185,14 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 - reference: '' - a1df63ad-0646-4dd2-a381-e1c9a6d2449b: !Template - answer_choices: Yes ||| No - id: a1df63ad-0646-4dd2-a381-e1c9a6d2449b - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' - reference: '' - a5cc9d03-4fc7-4d6e-9504-57e29c8f64d5: !Template - answer_choices: Yes ||| No - id: a5cc9d03-4fc7-4d6e-9504-57e29c8f64d5 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' + name: grammatical_between_1_2 reference: '' - b3f9fd42-1161-4530-8ad3-88e96a9997a7: !Template - answer_choices: Yes ||| No - id: b3f9fd42-1161-4530-8ad3-88e96a9997a7 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' - reference: '' - d2fd6631-2bc3-451b-9052-42514fb1826b: !Template - answer_choices: null - id: d2fd6631-2bc3-451b-9052-42514fb1826b + 951278d2-eb6f-4c55-9ec1-ebd7d351c465: !Template + answer_choices: A ||| B + id: 951278d2-eb6f-4c55-9ec1-ebd7d351c465 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -228,11 +212,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -240,42 +224,26 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 - reference: '' - e904fa30-f239-4b1c-82c6-f166b941a2cc: !Template - answer_choices: Yes ||| No - id: e904fa30-f239-4b1c-82c6-f166b941a2cc - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' + name: grammatical_which_one_A_B reference: '' - f6f7f7b3-dd21-4f16-bc7a-49d00278c807: !Template - answer_choices: null - id: f6f7f7b3-dd21-4f16-bc7a-49d00278c807 + ea1b4ca5-c215-4f29-92bd-f60b169b6ff2: !Template + answer_choices: 1 ||| 2 + id: ea1b4ca5-c215-4f29-92bd-f60b169b6ff2 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -283,11 +251,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -295,5 +263,37 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: grammatical_which_one_1_2 + reference: '' + f0817a39-597d-4fa5-8346-db551d9c5866: !Template + answer_choices: Yes ||| No + id: f0817a39-597d-4fa5-8346-db551d9c5866 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes + reference: '' + fcafa8ce-65ce-4b4d-8213-58c9f53558ec: !Template + answer_choices: Yes ||| No + id: fcafa8ce-65ce-4b4d-8213-58c9f53558ec + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes reference: '' diff --git a/promptsource/templates/blimp/determiner_noun_agreement_1/templates.yaml b/promptsource/templates/blimp/determiner_noun_agreement_1/templates.yaml index 60c7e2e08..60a88d368 100644 --- a/promptsource/templates/blimp/determiner_noun_agreement_1/templates.yaml +++ b/promptsource/templates/blimp/determiner_noun_agreement_1/templates.yaml @@ -1,9 +1,25 @@ dataset: blimp subset: determiner_noun_agreement_1 templates: - 00ea4c3a-48db-4033-b488-530f91b15f91: !Template + 01f50e2d-2102-4a35-a2cd-51d3b7592ddb: !Template + answer_choices: Yes ||| No + id: 01f50e2d-2102-4a35-a2cd-51d3b7592ddb + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 10ca8ed1-6c60-49c5-b8f3-2ae72288d97b: !Template answer_choices: A ||| B - id: 00ea4c3a-48db-4033-b488-530f91b15f91 + id: 10ca8ed1-6c60-49c5-b8f3-2ae72288d97b jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -37,46 +53,14 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: B-A)' - reference: '' - 2dffb9b8-a687-4ec2-9de4-2756bfccd6f8: !Template - answer_choices: Yes ||| No - id: 2dffb9b8-a687-4ec2-9de4-2756bfccd6f8 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' - reference: '' - 4eaa4f27-343b-46d3-ae1f-59506600c968: !Template - answer_choices: Yes ||| No - id: 4eaa4f27-343b-46d3-ae1f-59506600c968 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' + name: better_B_A reference: '' - 883c95e1-9c24-4dcc-b0a4-d36f791a5aec: !Template - answer_choices: null - id: 883c95e1-9c24-4dcc-b0a4-d36f791a5aec + 3729f629-d2a7-455e-8115-164873c05bfc: !Template + answer_choices: 1 ||| 2 + id: 3729f629-d2a7-455e-8115-164873c05bfc jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -96,11 +80,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -108,14 +92,46 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: grammatical_which_one_1_2 + reference: '' + 45c96cc3-56bd-41f3-a986-f61991465b11: !Template + answer_choices: Yes ||| No + id: 45c96cc3-56bd-41f3-a986-f61991465b11 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no reference: '' - 8eb177fa-7fce-4b25-8bbe-02a4e777d380: !Template - answer_choices: null - id: 8eb177fa-7fce-4b25-8bbe-02a4e777d380 + 519b861a-03a9-4f99-a832-9d9992b34116: !Template + answer_choices: Yes ||| No + id: 519b861a-03a9-4f99-a832-9d9992b34116 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes + reference: '' + 8466dd95-2656-4061-9cfd-abe7d4e36771: !Template + answer_choices: A ||| B + id: 8466dd95-2656-4061-9cfd-abe7d4e36771 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -135,11 +151,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -147,11 +163,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: grammatical_which_one_A_B reference: '' - 91c729c1-56e1-48f0-9252-912abe3ee7bf: !Template + 9d7f5a05-b76f-4ebc-93f2-194e438eab02: !Template answer_choices: A ||| B - id: 91c729c1-56e1-48f0-9252-912abe3ee7bf + id: 9d7f5a05-b76f-4ebc-93f2-194e438eab02 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -185,26 +201,26 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: A-B)' + name: better_A_B reference: '' - 9fe81394-ed07-434a-9525-0e7e5b9936c6: !Template - answer_choices: null - id: 9fe81394-ed07-434a-9525-0e7e5b9936c6 + ba5f23a8-d37b-40cb-aa6b-2fcd72269b2f: !Template + answer_choices: A ||| B + id: ba5f23a8-d37b-40cb-aa6b-2fcd72269b2f jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -212,11 +228,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -224,11 +240,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: grammatical_between_A_B reference: '' - a0883421-f6bc-4242-9142-fdb8bbd49240: !Template + d16ebcab-6ba3-4b8c-b3d4-d2065fa56b0f: !Template answer_choices: Yes ||| No - id: a0883421-f6bc-4242-9142-fdb8bbd49240 + id: d16ebcab-6ba3-4b8c-b3d4-d2065fa56b0f jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -240,42 +256,26 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' - reference: '' - be33a953-fbdb-41b3-896b-f750b93e97be: !Template - answer_choices: Yes ||| No - id: be33a953-fbdb-41b3-896b-f750b93e97be - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' + name: single_sentence_good_no_yes reference: '' - d9cf1e1e-cf34-415a-9db7-e5fee2febdaa: !Template - answer_choices: null - id: d9cf1e1e-cf34-415a-9db7-e5fee2febdaa + ed5abdcf-8f64-4073-9f3a-6b06ad93c053: !Template + answer_choices: 1 ||| 2 + id: ed5abdcf-8f64-4073-9f3a-6b06ad93c053 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -283,11 +283,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -295,5 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: grammatical_between_1_2 reference: '' diff --git a/promptsource/templates/blimp/determiner_noun_agreement_2/templates.yaml b/promptsource/templates/blimp/determiner_noun_agreement_2/templates.yaml index 05f49dae4..33f9bb024 100644 --- a/promptsource/templates/blimp/determiner_noun_agreement_2/templates.yaml +++ b/promptsource/templates/blimp/determiner_noun_agreement_2/templates.yaml @@ -1,9 +1,9 @@ dataset: blimp subset: determiner_noun_agreement_2 templates: - 150b9c09-a4fd-410a-8481-3d29c22e0264: !Template - answer_choices: null - id: 150b9c09-a4fd-410a-8481-3d29c22e0264 + 0abbea35-91b3-479b-9aff-389b6cc2b9fe: !Template + answer_choices: 1 ||| 2 + id: 0abbea35-91b3-479b-9aff-389b6cc2b9fe jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -26,11 +26,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -38,11 +38,43 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: grammatical_between_1_2 + reference: '' + 155402ce-a653-413d-a9bc-e4f1c8d6e4f7: !Template + answer_choices: Yes ||| No + id: 155402ce-a653-413d-a9bc-e4f1c8d6e4f7 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 247f5291-8873-43cb-b523-0ee00bb56a69: !Template + answer_choices: Yes ||| No + id: 247f5291-8873-43cb-b523-0ee00bb56a69 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes reference: '' - 2926db8f-3363-4ecb-ad4d-04efaf2828bf: !Template + 33d6830a-d214-4ff8-80fd-553bdbedccfa: !Template answer_choices: A ||| B - id: 2926db8f-3363-4ecb-ad4d-04efaf2828bf + id: 33d6830a-d214-4ff8-80fd-553bdbedccfa jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -61,7 +93,7 @@ templates: {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order + Which one is a better sentence of English, A or B? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -76,46 +108,46 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: B-A)' + name: better_A_B reference: '' - 3de3ed27-eaf9-4d17-8853-57cec4ea5256: !Template + 3ea31cb4-bc35-42fe-80f7-4125ba58b309: !Template answer_choices: Yes ||| No - id: 3de3ed27-eaf9-4d17-8853-57cec4ea5256 + id: 3ea31cb4-bc35-42fe-80f7-4125ba58b309 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' + name: single_sentence_good_yes_no reference: '' - 5e459dcd-b32e-4be2-808f-9a33418029aa: !Template + 5547dba3-b482-427b-9fbe-8491bdca060a: !Template answer_choices: Yes ||| No - id: 5e459dcd-b32e-4be2-808f-9a33418029aa + id: 5547dba3-b482-427b-9fbe-8491bdca060a jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' + name: single_sentence_bad_no_yes reference: '' - 80accaec-5e37-4f7d-ac1b-51f7f4fc6d51: !Template - answer_choices: null - id: 80accaec-5e37-4f7d-ac1b-51f7f4fc6d51 + 60f34769-c923-419c-bd42-fc289201eace: !Template + answer_choices: A ||| B + id: 60f34769-c923-419c-bd42-fc289201eace jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -135,11 +167,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -147,26 +179,26 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: grammatical_between_A_B reference: '' - 986c414b-489b-416c-bdf1-14edbea3bbec: !Template - answer_choices: null - id: 986c414b-489b-416c-bdf1-14edbea3bbec + 6ed8dd47-f3bf-4365-a0ad-2dde1bace547: !Template + answer_choices: 1 ||| 2 + id: 6ed8dd47-f3bf-4365-a0ad-2dde1bace547 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -174,11 +206,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -186,26 +218,26 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: grammatical_which_one_1_2 reference: '' - 9b737e07-3c3c-4c79-8ec8-0f68abd102b1: !Template - answer_choices: null - id: 9b737e07-3c3c-4c79-8ec8-0f68abd102b1 + 95573879-a945-41c4-9a02-e256879622b9: !Template + answer_choices: A ||| B + id: 95573879-a945-41c4-9a02-e256879622b9 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -213,11 +245,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -225,43 +257,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 - reference: '' - a041fb4e-2406-48f9-8058-a8ad1b2629dd: !Template - answer_choices: Yes ||| No - id: a041fb4e-2406-48f9-8058-a8ad1b2629dd - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' - reference: '' - b1a6eefc-c595-4ce4-8c29-d22303630a9c: !Template - answer_choices: Yes ||| No - id: b1a6eefc-c595-4ce4-8c29-d22303630a9c - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' + name: grammatical_which_one_A_B reference: '' - d915a001-fed4-4d8a-8ed3-319d2ecee41e: !Template + a610bc03-9da1-482e-93bc-948c5bbc560d: !Template answer_choices: A ||| B - id: d915a001-fed4-4d8a-8ed3-319d2ecee41e + id: a610bc03-9da1-482e-93bc-948c5bbc560d jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -280,7 +280,7 @@ templates: {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order + Which one is a better sentence of English, B or A? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -295,5 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: A-B)' + name: better_B_A reference: '' diff --git a/promptsource/templates/blimp/determiner_noun_agreement_irregular_1/templates.yaml b/promptsource/templates/blimp/determiner_noun_agreement_irregular_1/templates.yaml index bb62b3313..add4dae6e 100644 --- a/promptsource/templates/blimp/determiner_noun_agreement_irregular_1/templates.yaml +++ b/promptsource/templates/blimp/determiner_noun_agreement_irregular_1/templates.yaml @@ -1,36 +1,35 @@ dataset: blimp subset: determiner_noun_agreement_irregular_1 templates: - 1d5e79e9-fc6f-4a0f-9b99-06259c556d8f: !Template - answer_choices: null - id: 1d5e79e9-fc6f-4a0f-9b99-06259c556d8f + 0be7739d-5820-471d-ab43-14e5ea6baee9: !Template + answer_choices: A ||| B + id: 0be7739d-5820-471d-ab43-14e5ea6baee9 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -38,27 +37,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B - reference: '' - 3107c8b7-a4f3-4083-a33b-32e676b8a7ce: !Template - answer_choices: Yes ||| No - id: 3107c8b7-a4f3-4083-a33b-32e676b8a7ce - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' + name: better_B_A reference: '' - 34c18d68-dc3d-442d-a942-06bccfd2ed07: !Template + 3df7c754-c0f9-403b-9dec-9de4b0d08b10: !Template answer_choices: Yes ||| No - id: 34c18d68-dc3d-442d-a942-06bccfd2ed07 + id: 3df7c754-c0f9-403b-9dec-9de4b0d08b10 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -70,11 +53,11 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' + name: single_sentence_good_no_yes reference: '' - 51bd6ce0-0d94-47bc-b64a-2f15182084e3: !Template - answer_choices: null - id: 51bd6ce0-0d94-47bc-b64a-2f15182084e3 + 4a936cf9-76f9-4905-9bb0-a2e501fc26d3: !Template + answer_choices: 1 ||| 2 + id: 4a936cf9-76f9-4905-9bb0-a2e501fc26d3 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer 1 or 2. @@ -97,11 +80,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -109,38 +92,37 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: grammatical_which_one_1_2 reference: '' - 5388a8c8-ac53-4b26-8253-5878d34ea323: !Template - answer_choices: null - id: 5388a8c8-ac53-4b26-8253-5878d34ea323 + 639e489a-2d99-44d8-9d88-a652437ecda7: !Template + answer_choices: A ||| B + id: 639e489a-2d99-44d8-9d88-a652437ecda7 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -148,47 +130,32 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 - reference: '' - 90d6a1bc-fab5-43aa-be78-157d39ca7de1: !Template - answer_choices: Yes ||| No - id: 90d6a1bc-fab5-43aa-be78-157d39ca7de1 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' + name: better_A_B reference: '' - aa090d09-d35e-4d79-8378-941efa1c96e7: !Template - answer_choices: A ||| B - id: aa090d09-d35e-4d79-8378-941efa1c96e7 + 7367f407-4707-4b75-91a7-6a49fa0faffc: !Template + answer_choices: 1 ||| 2 + id: 7367f407-4707-4b75-91a7-6a49fa0faffc jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -202,31 +169,80 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: B-A)' + name: grammatical_between_1_2 reference: '' - bd74eaa5-1999-48ee-9e49-b7638ca17d0e: !Template + aaff53b9-8a57-433d-a7a5-c4935a8ca799: !Template + answer_choices: Yes ||| No + id: aaff53b9-8a57-433d-a7a5-c4935a8ca799 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes + reference: '' + bf3f6215-b44d-42a2-a0ba-ab139d18bd06: !Template + answer_choices: Yes ||| No + id: bf3f6215-b44d-42a2-a0ba-ab139d18bd06 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + c26344c9-cc1e-4820-9bb8-426583791567: !Template + answer_choices: Yes ||| No + id: c26344c9-cc1e-4820-9bb8-426583791567 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + c2959e6a-353a-4195-99e6-1dd49e1046ba: !Template answer_choices: A ||| B - id: bd74eaa5-1999-48ee-9e49-b7638ca17d0e + id: c2959e6a-353a-4195-99e6-1dd49e1046ba jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -240,26 +256,26 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: A-B)' + name: grammatical_which_one_A_B reference: '' - e4511bf0-e90e-4083-94e4-eeddffce3107: !Template - answer_choices: null - id: e4511bf0-e90e-4083-94e4-eeddffce3107 + e6c55627-465f-40f4-bae6-248afeced026: !Template + answer_choices: A ||| B + id: e6c55627-465f-40f4-bae6-248afeced026 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -267,11 +283,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -279,21 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 - reference: '' - f61ddf80-f696-45e8-abe1-583cd2caf963: !Template - answer_choices: Yes ||| No - id: f61ddf80-f696-45e8-abe1-583cd2caf963 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' + name: grammatical_between_A_B reference: '' diff --git a/promptsource/templates/blimp/determiner_noun_agreement_irregular_2/templates.yaml b/promptsource/templates/blimp/determiner_noun_agreement_irregular_2/templates.yaml index a6a4c0783..0506ba69e 100644 --- a/promptsource/templates/blimp/determiner_noun_agreement_irregular_2/templates.yaml +++ b/promptsource/templates/blimp/determiner_noun_agreement_irregular_2/templates.yaml @@ -1,24 +1,40 @@ dataset: blimp subset: determiner_noun_agreement_irregular_2 templates: - 12d29269-013c-428b-bfac-112a9bcfcee0: !Template - answer_choices: null - id: 12d29269-013c-428b-bfac-112a9bcfcee0 + 26515490-1232-4b88-abd0-26e6e9d5fa56: !Template + answer_choices: Yes ||| No + id: 26515490-1232-4b88-abd0-26e6e9d5fa56 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 4566b72e-cfa9-4cc9-ade5-3164d9740d75: !Template + answer_choices: 1 ||| 2 + id: 4566b72e-cfa9-4cc9-ade5-3164d9740d75 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -26,11 +42,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -38,26 +54,26 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: grammatical_which_one_1_2 reference: '' - 1ab8247a-a0cd-4efb-862e-ba4439c6f649: !Template - answer_choices: null - id: 1ab8247a-a0cd-4efb-862e-ba4439c6f649 + 4f5e375d-1f8d-432e-b9f9-a2f0eda9cc3e: !Template + answer_choices: A ||| B + id: 4f5e375d-1f8d-432e-b9f9-a2f0eda9cc3e jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -65,11 +81,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -77,38 +93,37 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: grammatical_between_A_B reference: '' - 41da01a4-1c38-4399-9e99-2d9b15811aeb: !Template - answer_choices: null - id: 41da01a4-1c38-4399-9e99-2d9b15811aeb + 5043d1cb-4265-4cec-9510-d95f0dca1a31: !Template + answer_choices: A ||| B + id: 5043d1cb-4265-4cec-9510-d95f0dca1a31 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -116,30 +131,14 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 - reference: '' - 51876660-1c80-4607-a6af-63461ed4172a: !Template - answer_choices: Yes ||| No - id: 51876660-1c80-4607-a6af-63461ed4172a - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' + name: better_A_B reference: '' - 52c1851e-b0b8-4c09-9e30-9b921b3d0059: !Template - answer_choices: null - id: 52c1851e-b0b8-4c09-9e30-9b921b3d0059 + 60b2cf66-c88b-4e08-9f9d-6f25134e0be3: !Template + answer_choices: A ||| B + id: 60b2cf66-c88b-4e08-9f9d-6f25134e0be3 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -159,11 +158,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -171,31 +170,48 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: grammatical_which_one_A_B reference: '' - 60db8bc5-52f7-4155-b5b4-cc8007c166c6: !Template - answer_choices: A ||| B - id: 60db8bc5-52f7-4155-b5b4-cc8007c166c6 + 6f982cc1-127e-4ba0-b757-4c7f0d3b1c48: !Template + answer_choices: Yes ||| No + id: 6f982cc1-127e-4ba0-b757-4c7f0d3b1c48 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 7d2de496-10ea-47fb-b38f-44a6856d3764: !Template + answer_choices: 1 ||| 2 + id: 7d2de496-10ea-47fb-b38f-44a6856d3764 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -209,11 +225,27 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: B-A)' + name: grammatical_between_1_2 + reference: '' + ab9d74f5-4504-41bc-b3b4-cf2c624f1fe2: !Template + answer_choices: Yes ||| No + id: ab9d74f5-4504-41bc-b3b4-cf2c624f1fe2 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes reference: '' - 697420f0-4e22-45bd-b182-8f7e774a52dd: !Template + b05688a5-ca11-4bcd-af6f-b9d21ff14294: !Template answer_choices: A ||| B - id: 697420f0-4e22-45bd-b182-8f7e774a52dd + id: b05688a5-ca11-4bcd-af6f-b9d21ff14294 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -232,7 +264,7 @@ templates: {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order + Which one is a better sentence of English, B or A? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -247,11 +279,11 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: A-B)' + name: better_B_A reference: '' - a7a653e1-8764-4667-9af5-7f683acc5ad0: !Template + f990337c-829d-43d1-91a1-87e9bff30b7a: !Template answer_choices: Yes ||| No - id: a7a653e1-8764-4667-9af5-7f683acc5ad0 + id: f990337c-829d-43d1-91a1-87e9bff30b7a jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -263,37 +295,5 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' - reference: '' - b42a7a6c-efee-481a-b623-c45f27e60328: !Template - answer_choices: Yes ||| No - id: b42a7a6c-efee-481a-b623-c45f27e60328 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' - reference: '' - e8ca15b8-fbbb-49bd-b12a-80fee5a828c4: !Template - answer_choices: Yes ||| No - id: e8ca15b8-fbbb-49bd-b12a-80fee5a828c4 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' + name: single_sentence_good_no_yes reference: '' diff --git a/promptsource/templates/blimp/determiner_noun_agreement_with_adj_2/templates.yaml b/promptsource/templates/blimp/determiner_noun_agreement_with_adj_2/templates.yaml index b4ac0e667..8a69962c7 100644 --- a/promptsource/templates/blimp/determiner_noun_agreement_with_adj_2/templates.yaml +++ b/promptsource/templates/blimp/determiner_noun_agreement_with_adj_2/templates.yaml @@ -1,64 +1,25 @@ dataset: blimp subset: determiner_noun_agreement_with_adj_2 templates: - 10b55fcc-3e50-4aac-9ff4-999e20c95892: !Template - answer_choices: null - id: 10b55fcc-3e50-4aac-9ff4-999e20c95892 - jinja: '{% set shuffled_order = [0, 1] | random %} - - Between the two sentences, which one is grammatical? Please answer A or B. - - {% if shuffled_order == 0 %} - - A: {{ sentence_good }} - - B: {{ sentence_bad }} - - {% else %} - - A: {{ sentence_bad }} - - B: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - A - - {% else %} - - B - - {% endif %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: random_choice_A_B_2 - reference: '' - 13ad95ed-9249-464e-ad40-6aba1dc8dfbf: !Template + 2702ea88-ffe0-4176-b033-1fb6b022b0f2: !Template answer_choices: Yes ||| No - id: 13ad95ed-9249-464e-ad40-6aba1dc8dfbf + id: 2702ea88-ffe0-4176-b033-1fb6b022b0f2 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' + name: single_sentence_good_yes_no reference: '' - 31efe64e-08b8-45fa-9685-9fedb1d974cf: !Template - answer_choices: null - id: 31efe64e-08b8-45fa-9685-9fedb1d974cf + 4306aa5e-6363-4853-aa80-2504a802930c: !Template + answer_choices: 1 ||| 2 + id: 4306aa5e-6363-4853-aa80-2504a802930c jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer 1 or 2. @@ -81,11 +42,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -93,26 +54,26 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: grammatical_which_one_1_2 reference: '' - 40cb218a-e034-4292-a4cd-b1ec05f16889: !Template - answer_choices: null - id: 40cb218a-e034-4292-a4cd-b1ec05f16889 + 45eba5a5-d68d-4587-a641-ff398dc9005f: !Template + answer_choices: A ||| B + id: 45eba5a5-d68d-4587-a641-ff398dc9005f jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -120,11 +81,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -132,14 +93,14 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: grammatical_which_one_A_B reference: '' - 79cdbcef-1785-4f19-84f5-91507a1f0c9d: !Template - answer_choices: null - id: 79cdbcef-1785-4f19-84f5-91507a1f0c9d + 64c92ead-51cd-43da-94bb-54000b7e8798: !Template + answer_choices: A ||| B + id: 64c92ead-51cd-43da-94bb-54000b7e8798 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -159,11 +120,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -171,31 +132,53 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: grammatical_between_A_B reference: '' - 7f94d9c9-acb2-4ad1-b78d-3dafa7f7ce8e: !Template - answer_choices: Yes ||| No - id: 7f94d9c9-acb2-4ad1-b78d-3dafa7f7ce8e - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + 73d3e906-e9af-4f24-a9cb-128dd81b7dd9: !Template + answer_choices: A ||| B + id: 73d3e906-e9af-4f24-a9cb-128dd81b7dd9 + jinja: '{% set shuffled_order = [0, 1] | random %} - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' + original_task: true + name: better_B_A reference: '' - 89bd2199-5b38-4269-aeb8-047e12d5970e: !Template + 89179eb8-24f0-44f7-a603-dc44db267b9c: !Template answer_choices: Yes ||| No - id: 89bd2199-5b38-4269-aeb8-047e12d5970e + id: 89179eb8-24f0-44f7-a603-dc44db267b9c jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -203,47 +186,48 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' + name: single_sentence_good_no_yes reference: '' - c2c457a7-2e46-41a9-b2f2-2838855552dc: !Template + 89dd1889-2341-42c8-9354-47aa9747da7b: !Template answer_choices: Yes ||| No - id: c2c457a7-2e46-41a9-b2f2-2838855552dc + id: 89dd1889-2341-42c8-9354-47aa9747da7b jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' + name: single_sentence_bad_no_yes reference: '' - cfa06ff8-02bd-4a1b-8737-720010838a62: !Template - answer_choices: A ||| B - id: cfa06ff8-02bd-4a1b-8737-720010838a62 + d5dc95bc-088d-46e7-a22f-e34f038af42e: !Template + answer_choices: 1 ||| 2 + id: d5dc95bc-088d-46e7-a22f-e34f038af42e jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -257,11 +241,11 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: A-B)' + name: grammatical_between_1_2 reference: '' - d637a459-6f73-4a5d-9b16-cc34ca66ecd3: !Template + dcbe84c8-fe8f-450f-8f70-4bf34d21b6a4: !Template answer_choices: A ||| B - id: d637a459-6f73-4a5d-9b16-cc34ca66ecd3 + id: dcbe84c8-fe8f-450f-8f70-4bf34d21b6a4 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -280,7 +264,7 @@ templates: {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order + Which one is a better sentence of English, A or B? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -295,5 +279,21 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: B-A)' + name: better_A_B + reference: '' + ea091a0b-0ec2-4a44-aaeb-3a6d1772e477: !Template + answer_choices: Yes ||| No + id: ea091a0b-0ec2-4a44-aaeb-3a6d1772e477 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no reference: '' diff --git a/promptsource/templates/blimp/determiner_noun_agreement_with_adj_irregular_1/templates.yaml b/promptsource/templates/blimp/determiner_noun_agreement_with_adj_irregular_1/templates.yaml index 79d3c5809..66f7a647b 100644 --- a/promptsource/templates/blimp/determiner_noun_agreement_with_adj_irregular_1/templates.yaml +++ b/promptsource/templates/blimp/determiner_noun_agreement_with_adj_irregular_1/templates.yaml @@ -1,52 +1,35 @@ dataset: blimp subset: determiner_noun_agreement_with_adj_irregular_1 templates: - 045f504c-76d8-4225-973b-6d835118bd92: !Template - answer_choices: Yes ||| No - id: 045f504c-76d8-4225-973b-6d835118bd92 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' - reference: '' - 186beef9-5742-42d8-b75d-c011ad3fc4c5: !Template - answer_choices: null - id: 186beef9-5742-42d8-b75d-c011ad3fc4c5 + 1dce2f7c-1341-497b-b824-c24120fd66d7: !Template + answer_choices: A ||| B + id: 1dce2f7c-1341-497b-b824-c24120fd66d7 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -54,31 +37,32 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: better_A_B reference: '' - 6288f326-e84d-4149-83eb-0035ece04ff4: !Template - answer_choices: A ||| B - id: 6288f326-e84d-4149-83eb-0035ece04ff4 + 2339eda9-fc03-4207-a71b-a5cc01b2e13a: !Template + answer_choices: 1 ||| 2 + id: 2339eda9-fc03-4207-a71b-a5cc01b2e13a jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -92,11 +76,11 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: B-A)' + name: grammatical_between_1_2 reference: '' - 7a9598d2-8dc3-40fd-a143-033c9cb133c8: !Template + 38c55b36-426a-4001-86dd-391c643adab5: !Template answer_choices: A ||| B - id: 7a9598d2-8dc3-40fd-a143-033c9cb133c8 + id: 38c55b36-426a-4001-86dd-391c643adab5 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -115,7 +99,7 @@ templates: {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order + Which one is a better sentence of English, B or A? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -130,14 +114,46 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: A-B)' + name: better_B_A + reference: '' + 47e22a97-df75-41c9-b1f1-28cf9c99ea91: !Template + answer_choices: Yes ||| No + id: 47e22a97-df75-41c9-b1f1-28cf9c99ea91 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 59f3abfa-7cf2-48a5-9c0c-bc5467e8c6d9: !Template + answer_choices: Yes ||| No + id: 59f3abfa-7cf2-48a5-9c0c-bc5467e8c6d9 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no reference: '' - 7c10eaa6-8c76-418d-86de-4ed59c45cdb3: !Template - answer_choices: null - id: 7c10eaa6-8c76-418d-86de-4ed59c45cdb3 + 5ace59df-aa43-479b-b2eb-ea2b7d1204ba: !Template + answer_choices: A ||| B + id: 5ace59df-aa43-479b-b2eb-ea2b7d1204ba jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -157,11 +173,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -169,26 +185,42 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: grammatical_between_A_B + reference: '' + 6d08e8d4-769f-4e1c-b436-f5663f0c9a54: !Template + answer_choices: Yes ||| No + id: 6d08e8d4-769f-4e1c-b436-f5663f0c9a54 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes reference: '' - 86a56515-55d5-4ecc-89bd-88b17b885b3d: !Template - answer_choices: null - id: 86a56515-55d5-4ecc-89bd-88b17b885b3d + b8e57aff-fbb4-4016-afba-048ec42f6ebf: !Template + answer_choices: 1 ||| 2 + id: b8e57aff-fbb4-4016-afba-048ec42f6ebf jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -196,11 +228,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -208,11 +240,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: grammatical_which_one_1_2 reference: '' - 95597f80-2588-492c-8972-db0c61d4f105: !Template + e541b31d-9263-4dff-86d1-d536f367c42f: !Template answer_choices: Yes ||| No - id: 95597f80-2588-492c-8972-db0c61d4f105 + id: e541b31d-9263-4dff-86d1-d536f367c42f jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -224,42 +256,26 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' - reference: '' - 9e886594-52fe-474f-94c3-f6c4dee8f398: !Template - answer_choices: Yes ||| No - id: 9e886594-52fe-474f-94c3-f6c4dee8f398 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' + name: single_sentence_good_no_yes reference: '' - a9207b62-c320-4444-b5f6-75cd7a8a4879: !Template - answer_choices: null - id: a9207b62-c320-4444-b5f6-75cd7a8a4879 + f8d2ddd5-b969-4086-9bf0-a7fa11020a13: !Template + answer_choices: A ||| B + id: f8d2ddd5-b969-4086-9bf0-a7fa11020a13 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -267,11 +283,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -279,21 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 - reference: '' - ff4ec272-1749-4ea5-a522-d9312fb6569d: !Template - answer_choices: Yes ||| No - id: ff4ec272-1749-4ea5-a522-d9312fb6569d - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' + name: grammatical_which_one_A_B reference: '' diff --git a/promptsource/templates/blimp/determiner_noun_agreement_with_adj_irregular_2/templates.yaml b/promptsource/templates/blimp/determiner_noun_agreement_with_adj_irregular_2/templates.yaml index 1bfb81fce..4fe0fdbbb 100644 --- a/promptsource/templates/blimp/determiner_noun_agreement_with_adj_irregular_2/templates.yaml +++ b/promptsource/templates/blimp/determiner_noun_agreement_with_adj_irregular_2/templates.yaml @@ -1,24 +1,24 @@ dataset: blimp subset: determiner_noun_agreement_with_adj_irregular_2 templates: - 39d35024-fd56-4b59-9890-4cf7ef67c55f: !Template - answer_choices: null - id: 39d35024-fd56-4b59-9890-4cf7ef67c55f + 1e5ff764-2666-4a46-bdf7-bbc53e945186: !Template + answer_choices: 1 ||| 2 + id: 1e5ff764-2666-4a46-bdf7-bbc53e945186 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -26,11 +26,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -38,47 +38,53 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: grammatical_between_1_2 reference: '' - 4bf11505-3a29-4861-ac4c-9bd9871361c7: !Template - answer_choices: Yes ||| No - id: 4bf11505-3a29-4861-ac4c-9bd9871361c7 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + 44ef80e5-725b-40cf-b7b7-db754c332682: !Template + answer_choices: A ||| B + id: 44ef80e5-725b-40cf-b7b7-db754c332682 + jinja: '{% set shuffled_order = [0, 1] | random %} - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' - reference: '' - 69ba54e3-d0e6-47c3-a1b9-184e7f8321bd: !Template - answer_choices: Yes ||| No - id: 69ba54e3-d0e6-47c3-a1b9-184e7f8321bd - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + Read the following two sentences. - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' + original_task: true + name: better_A_B reference: '' - 80dac6de-36f7-4cb8-a538-58e80d6a98e4: !Template + 53352573-715b-4152-a73c-0a9b58325c8f: !Template answer_choices: Yes ||| No - id: 80dac6de-36f7-4cb8-a538-58e80d6a98e4 + id: 53352573-715b-4152-a73c-0a9b58325c8f jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata @@ -86,11 +92,11 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' + name: single_sentence_bad_no_yes reference: '' - 89ff4368-1271-4496-abed-312e101fe8b1: !Template - answer_choices: null - id: 89ff4368-1271-4496-abed-312e101fe8b1 + 6afacf5a-d598-426f-8b82-bcb6a6ab5ee5: !Template + answer_choices: A ||| B + id: 6afacf5a-d598-426f-8b82-bcb6a6ab5ee5 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer A or B. @@ -113,11 +119,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -125,50 +131,27 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: grammatical_between_A_B reference: '' - a16e57d6-14b1-40d6-a9f7-e24349e46246: !Template - answer_choices: null - id: a16e57d6-14b1-40d6-a9f7-e24349e46246 - jinja: '{% set shuffled_order = [0, 1] | random %} - - Between the two sentences, which one is grammatical? Please answer 1 or 2. - - {% if shuffled_order == 0 %} - - 1: {{ sentence_good }} - - 2: {{ sentence_bad }} - - {% else %} - - 1: {{ sentence_bad }} - - 2: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - 1 - - {% else %} - - 2 + 8b2223e4-1a76-4bb1-8856-5aeb8b0ae838: !Template + answer_choices: Yes ||| No + id: 8b2223e4-1a76-4bb1-8856-5aeb8b0ae838 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - {% endif %}' + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: random_choice_1_2_2 + original_task: false + name: single_sentence_good_yes_no reference: '' - b5b0072c-fbef-4cd0-91b7-e52c2db5b561: !Template - answer_choices: null - id: b5b0072c-fbef-4cd0-91b7-e52c2db5b561 + 94b67102-49c3-41d1-9c85-08043921617c: !Template + answer_choices: 1 ||| 2 + id: 94b67102-49c3-41d1-9c85-08043921617c jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer 1 or 2. @@ -191,11 +174,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -203,11 +186,27 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: grammatical_which_one_1_2 reference: '' - c14756c3-1e8e-4982-a3a3-20495198880f: !Template + a3562e15-369b-4fa2-919d-e8dfe5b99df6: !Template + answer_choices: Yes ||| No + id: a3562e15-369b-4fa2-919d-e8dfe5b99df6 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + c11f15e2-1678-440d-a6ba-ed4630942049: !Template answer_choices: A ||| B - id: c14756c3-1e8e-4982-a3a3-20495198880f + id: c11f15e2-1678-440d-a6ba-ed4630942049 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -226,7 +225,7 @@ templates: {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order + Which one is a better sentence of English, B or A? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -241,31 +240,48 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: A-B)' + name: better_B_A + reference: '' + c1cd46ac-c799-4f28-a72c-e51d4a02a5b5: !Template + answer_choices: Yes ||| No + id: c1cd46ac-c799-4f28-a72c-e51d4a02a5b5 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes reference: '' - f1d757a3-d31a-4c19-a23f-73a6d9cfdbb0: !Template + e6215cbd-8dde-46b3-b79b-b869eb7f2c4b: !Template answer_choices: A ||| B - id: f1d757a3-d31a-4c19-a23f-73a6d9cfdbb0 + id: e6215cbd-8dde-46b3-b79b-b869eb7f2c4b jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -279,21 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: B-A)' - reference: '' - fee5c349-3bf7-4bc9-b190-fdf2c0c27c3d: !Template - answer_choices: Yes ||| No - id: fee5c349-3bf7-4bc9-b190-fdf2c0c27c3d - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' + name: grammatical_which_one_A_B reference: '' diff --git a/promptsource/templates/blimp/determiner_noun_agreement_with_adjective_1/templates.yaml b/promptsource/templates/blimp/determiner_noun_agreement_with_adjective_1/templates.yaml index 4cf86557d..37945da63 100644 --- a/promptsource/templates/blimp/determiner_noun_agreement_with_adjective_1/templates.yaml +++ b/promptsource/templates/blimp/determiner_noun_agreement_with_adjective_1/templates.yaml @@ -1,13 +1,13 @@ dataset: blimp subset: determiner_noun_agreement_with_adjective_1 templates: - 02004ede-c0aa-4210-b96c-ad62c10ed818: !Template + 2b99daa2-9a73-44fb-b704-d28079ad0f3e: !Template answer_choices: Yes ||| No - id: 02004ede-c0aa-4210-b96c-ad62c10ed818 + id: 2b99daa2-9a73-44fb-b704-d28079ad0f3e jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -15,14 +15,30 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' + name: single_sentence_good_no_yes reference: '' - 11231ce2-c610-4fdc-96bd-8545bec4b174: !Template - answer_choices: null - id: 11231ce2-c610-4fdc-96bd-8545bec4b174 + 375d054d-246f-4ba2-af73-ec7b8a9d4681: !Template + answer_choices: Yes ||| No + id: 375d054d-246f-4ba2-af73-ec7b8a9d4681 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 3da7b8e5-ddd2-4099-8045-8b3ca57bd3e2: !Template + answer_choices: 1 ||| 2 + id: 3da7b8e5-ddd2-4099-8045-8b3ca57bd3e2 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -42,11 +58,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -54,30 +70,14 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 - reference: '' - 152f85f8-555b-4065-83e7-a11395d6d463: !Template - answer_choices: Yes ||| No - id: 152f85f8-555b-4065-83e7-a11395d6d463 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' + name: grammatical_which_one_1_2 reference: '' - 1c768f5c-9028-4f5e-a5e7-9a7ff83b548c: !Template - answer_choices: null - id: 1c768f5c-9028-4f5e-a5e7-9a7ff83b548c + 411946bc-ff32-41d0-afd9-0b9d9b87669d: !Template + answer_choices: 1 ||| 2 + id: 411946bc-ff32-41d0-afd9-0b9d9b87669d jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -97,11 +97,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -109,38 +109,37 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: grammatical_between_1_2 reference: '' - 51ffb0c5-488e-40a8-8048-4436b0962548: !Template - answer_choices: null - id: 51ffb0c5-488e-40a8-8048-4436b0962548 + 599dbe06-4c60-44e2-9fa0-63ce24c25191: !Template + answer_choices: A ||| B + id: 599dbe06-4c60-44e2-9fa0-63ce24c25191 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -148,15 +147,15 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: better_B_A reference: '' - 94f6be35-8f67-4298-abcd-ccaaca33ce1f: !Template + 688dd368-4fda-4442-8cf8-a9c324c6cdc4: !Template answer_choices: Yes ||| No - id: 94f6be35-8f67-4298-abcd-ccaaca33ce1f + id: 688dd368-4fda-4442-8cf8-a9c324c6cdc4 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -164,31 +163,32 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' + name: single_sentence_good_yes_no reference: '' - 9cc90e49-8d9b-486d-9ec3-15829173017e: !Template + 8542d02a-7270-4eaf-b9df-ce7d0a36eb5d: !Template answer_choices: A ||| B - id: 9cc90e49-8d9b-486d-9ec3-15829173017e + id: 8542d02a-7270-4eaf-b9df-ce7d0a36eb5d jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -202,27 +202,11 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: B-A)' - reference: '' - b5ec67eb-6356-474a-a54d-a94f2717983d: !Template - answer_choices: Yes ||| No - id: b5ec67eb-6356-474a-a54d-a94f2717983d - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' + name: grammatical_which_one_A_B reference: '' - d676ab08-308d-48f2-aab1-50211261475e: !Template + cf1c7c75-b761-453d-bdb6-8ad0b464b979: !Template answer_choices: A ||| B - id: d676ab08-308d-48f2-aab1-50211261475e + id: cf1c7c75-b761-453d-bdb6-8ad0b464b979 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -256,11 +240,11 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: A-B)' + name: better_A_B reference: '' - da246a25-3001-421f-86d6-e07b11281f06: !Template - answer_choices: null - id: da246a25-3001-421f-86d6-e07b11281f06 + d64b6f4a-1759-4c71-aeef-ea164c474bba: !Template + answer_choices: A ||| B + id: d64b6f4a-1759-4c71-aeef-ea164c474bba jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer A or B. @@ -283,11 +267,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -295,5 +279,21 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: grammatical_between_A_B + reference: '' + e1dc6aa2-7fb4-4fa7-bb49-e5af7dd2c298: !Template + answer_choices: Yes ||| No + id: e1dc6aa2-7fb4-4fa7-bb49-e5af7dd2c298 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes reference: '' diff --git a/promptsource/templates/blimp/distractor_agreement_relational_noun/templates.yaml b/promptsource/templates/blimp/distractor_agreement_relational_noun/templates.yaml index 88aa0090d..b71d50866 100644 --- a/promptsource/templates/blimp/distractor_agreement_relational_noun/templates.yaml +++ b/promptsource/templates/blimp/distractor_agreement_relational_noun/templates.yaml @@ -1,48 +1,25 @@ dataset: blimp subset: distractor_agreement_relational_noun templates: - 0093546a-ad1b-4211-99d7-32298a95a6c8: !Template - answer_choices: null - id: 0093546a-ad1b-4211-99d7-32298a95a6c8 - jinja: '{% set shuffled_order = [0, 1] | random %} - - Between the two sentences, which one is grammatical? Please answer A or B. - - {% if shuffled_order == 0 %} - - A: {{ sentence_good }} - - B: {{ sentence_bad }} - - {% else %} - - A: {{ sentence_bad }} - - B: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - A - - {% else %} - - B + 2c5fb585-9573-4112-9e49-64b7d480bec2: !Template + answer_choices: Yes ||| No + id: 2c5fb585-9573-4112-9e49-64b7d480bec2 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - {% endif %}' + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: random_choice_A_B_2 + original_task: false + name: single_sentence_good_yes_no reference: '' - 556132b2-dc63-4d0a-9979-bd36ec5adefb: !Template - answer_choices: null - id: 556132b2-dc63-4d0a-9979-bd36ec5adefb + 3e406e15-6f42-402a-8249-55362efe00c7: !Template + answer_choices: 1 ||| 2 + id: 3e406e15-6f42-402a-8249-55362efe00c7 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer 1 or 2. @@ -65,11 +42,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -77,31 +54,48 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: grammatical_which_one_1_2 reference: '' - 7d643f07-11f5-4948-8971-d2870478cb9a: !Template + 3eccd02a-3fe5-4aff-b579-63d698e8bada: !Template + answer_choices: Yes ||| No + id: 3eccd02a-3fe5-4aff-b579-63d698e8bada + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes + reference: '' + 605639df-072a-47c6-b166-41824a867688: !Template answer_choices: A ||| B - id: 7d643f07-11f5-4948-8971-d2870478cb9a + id: 605639df-072a-47c6-b166-41824a867688 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -115,58 +109,42 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: B-A)' - reference: '' - 7e62f966-8cfa-4e6b-b57b-39a6650ad434: !Template - answer_choices: Yes ||| No - id: 7e62f966-8cfa-4e6b-b57b-39a6650ad434 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' + name: grammatical_between_A_B reference: '' - 7f41d5b1-dbd2-4a6d-9383-6cf3e8340228: !Template + 8cf993bd-33db-41ee-92b3-1ed8132bdf84: !Template answer_choices: Yes ||| No - id: 7f41d5b1-dbd2-4a6d-9383-6cf3e8340228 + id: 8cf993bd-33db-41ee-92b3-1ed8132bdf84 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' + name: single_sentence_bad_no_yes reference: '' - 9bcba8ef-215d-4c53-8855-c7139ce89f87: !Template - answer_choices: null - id: 9bcba8ef-215d-4c53-8855-c7139ce89f87 + 91569423-f8be-4f41-a740-fbadb9a3966e: !Template + answer_choices: A ||| B + id: 91569423-f8be-4f41-a740-fbadb9a3966e jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -174,11 +152,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -186,38 +164,37 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: grammatical_which_one_A_B reference: '' - a50f9477-e887-47ab-86c3-25bbe13bde5f: !Template - answer_choices: null - id: a50f9477-e887-47ab-86c3-25bbe13bde5f + 9d474aca-b18a-4968-a5f5-dc16e4f79811: !Template + answer_choices: A ||| B + id: 9d474aca-b18a-4968-a5f5-dc16e4f79811 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -225,11 +202,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: better_A_B reference: '' - bf30aa91-05b9-4339-8608-d68470a1dec1: !Template + dfe23dc0-6a82-4c40-9f3a-8340bf6fa7f1: !Template answer_choices: A ||| B - id: bf30aa91-05b9-4339-8608-d68470a1dec1 + id: dfe23dc0-6a82-4c40-9f3a-8340bf6fa7f1 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -248,7 +225,7 @@ templates: {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order + Which one is a better sentence of English, B or A? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -263,11 +240,11 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: A-B)' + name: better_B_A reference: '' - bfcde61d-b884-44c2-8e17-6cd8f98af970: !Template + eb9d8608-b55c-468a-8db5-790e87e09bd9: !Template answer_choices: Yes ||| No - id: bfcde61d-b884-44c2-8e17-6cd8f98af970 + id: eb9d8608-b55c-468a-8db5-790e87e09bd9 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -279,21 +256,44 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' + name: single_sentence_bad_yes_no reference: '' - ddb15e2e-07ec-401a-bd45-494f2f182103: !Template - answer_choices: Yes ||| No - id: ddb15e2e-07ec-401a-bd45-494f2f182103 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + fd46b0be-1134-4cef-8ec3-a8807365683d: !Template + answer_choices: 1 ||| 2 + id: fd46b0be-1134-4cef-8ec3-a8807365683d + jinja: '{% set shuffled_order = [0, 1] | random %} - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' + original_task: true + name: grammatical_between_1_2 reference: '' diff --git a/promptsource/templates/blimp/distractor_agreement_relative_clause/templates.yaml b/promptsource/templates/blimp/distractor_agreement_relative_clause/templates.yaml index fdcd29a91..1e525b66e 100644 --- a/promptsource/templates/blimp/distractor_agreement_relative_clause/templates.yaml +++ b/promptsource/templates/blimp/distractor_agreement_relative_clause/templates.yaml @@ -1,29 +1,46 @@ dataset: blimp subset: distractor_agreement_relative_clause templates: - 04107f2b-4ab2-4403-a3fd-ed945870209c: !Template - answer_choices: A ||| B - id: 04107f2b-4ab2-4403-a3fd-ed945870209c + 2a088be4-6e1b-420d-9668-e38584fce3df: !Template + answer_choices: Yes ||| No + id: 2a088be4-6e1b-420d-9668-e38584fce3df + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 4c299217-6509-447e-80ee-c5834f0a18c4: !Template + answer_choices: 1 ||| 2 + id: 4c299217-6509-447e-80ee-c5834f0a18c4 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -37,11 +54,11 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: B-A)' + name: grammatical_between_1_2 reference: '' - 0f9a8483-1559-41ee-ab1b-db7ae5255d2b: !Template + 55390108-07d1-4776-b73b-8ea179138065: !Template answer_choices: Yes ||| No - id: 0f9a8483-1559-41ee-ab1b-db7ae5255d2b + id: 55390108-07d1-4776-b73b-8ea179138065 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -53,38 +70,53 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' + name: single_sentence_good_yes_no reference: '' - 24371907-ca39-4ee3-957b-a6391cb2a50a: !Template - answer_choices: null - id: 24371907-ca39-4ee3-957b-a6391cb2a50a + 87ea3119-81d0-4473-8e19-67c5e411670e: !Template + answer_choices: Yes ||| No + id: 87ea3119-81d0-4473-8e19-67c5e411670e + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes + reference: '' + 88793be0-3ce6-498a-a96e-a584e2dd0710: !Template + answer_choices: A ||| B + id: 88793be0-3ce6-498a-a96e-a584e2dd0710 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -92,26 +124,42 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: better_A_B + reference: '' + 9b0fe4e4-bbc5-4300-9b4c-f2c43928a009: !Template + answer_choices: Yes ||| No + id: 9b0fe4e4-bbc5-4300-9b4c-f2c43928a009 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes reference: '' - 2f019572-dfb6-4c6f-adfa-3cd5764e90e2: !Template - answer_choices: null - id: 2f019572-dfb6-4c6f-adfa-3cd5764e90e2 + a48b6cdf-67b5-42ff-9a27-f672c061f6ee: !Template + answer_choices: 1 ||| 2 + id: a48b6cdf-67b5-42ff-9a27-f672c061f6ee jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -119,11 +167,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -131,38 +179,37 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: grammatical_which_one_1_2 reference: '' - 604ea06e-a8b5-47d0-9fcf-43ac57a5ad9d: !Template - answer_choices: null - id: 604ea06e-a8b5-47d0-9fcf-43ac57a5ad9d + c31818c9-00e8-4c3c-835a-64ab19feecae: !Template + answer_choices: A ||| B + id: c31818c9-00e8-4c3c-835a-64ab19feecae jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -170,42 +217,26 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: better_B_A reference: '' - ca808663-bc8e-453b-81a0-bcc48e0612e1: !Template - answer_choices: Yes ||| No - id: ca808663-bc8e-453b-81a0-bcc48e0612e1 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' - reference: '' - cb995c35-e892-44ba-86bd-51c5e5be9cae: !Template - answer_choices: null - id: cb995c35-e892-44ba-86bd-51c5e5be9cae + f285dbe1-a96c-45d3-9efa-569e5e43d09c: !Template + answer_choices: A ||| B + id: f285dbe1-a96c-45d3-9efa-569e5e43d09c jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -213,11 +244,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -225,31 +256,32 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: grammatical_between_A_B reference: '' - e06e1a13-8350-4b80-8ef2-7b7f16ccf8e2: !Template + f506a776-19d1-4848-b311-257c2eeb1d36: !Template answer_choices: A ||| B - id: e06e1a13-8350-4b80-8ef2-7b7f16ccf8e2 + id: f506a776-19d1-4848-b311-257c2eeb1d36 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -263,37 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: A-B)' - reference: '' - e9f2ca2b-59f3-4150-b6fc-54d27b3fdb91: !Template - answer_choices: Yes ||| No - id: e9f2ca2b-59f3-4150-b6fc-54d27b3fdb91 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' - reference: '' - f0376181-8db5-46c2-ada3-d7333545f1f4: !Template - answer_choices: Yes ||| No - id: f0376181-8db5-46c2-ada3-d7333545f1f4 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' + name: grammatical_which_one_A_B reference: '' diff --git a/promptsource/templates/blimp/drop_argument/templates.yaml b/promptsource/templates/blimp/drop_argument/templates.yaml index 6470f6706..20923279f 100644 --- a/promptsource/templates/blimp/drop_argument/templates.yaml +++ b/promptsource/templates/blimp/drop_argument/templates.yaml @@ -1,9 +1,25 @@ dataset: blimp subset: drop_argument templates: - 0a28fdc6-e122-4146-99f9-e2dbe41129d0: !Template - answer_choices: null - id: 0a28fdc6-e122-4146-99f9-e2dbe41129d0 + 1de3a26c-989e-4842-a1b4-636a5d1422cc: !Template + answer_choices: Yes ||| No + id: 1de3a26c-989e-4842-a1b4-636a5d1422cc + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 3cd42d1c-5cd2-4b9b-be35-5ed27fc2aabc: !Template + answer_choices: 1 ||| 2 + id: 3cd42d1c-5cd2-4b9b-be35-5ed27fc2aabc jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer 1 or 2. @@ -26,11 +42,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -38,14 +54,14 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: grammatical_which_one_1_2 reference: '' - 0cf5e865-8718-4c83-a3cc-ceaade91a933: !Template - answer_choices: null - id: 0cf5e865-8718-4c83-a3cc-ceaade91a933 + 4ee226bc-2b51-4cbe-8ee2-a54a376413ff: !Template + answer_choices: A ||| B + id: 4ee226bc-2b51-4cbe-8ee2-a54a376413ff jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -65,11 +81,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -77,43 +93,27 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B - reference: '' - 117b47a4-0b41-4d0e-8dc1-2c2eb76befda: !Template - answer_choices: Yes ||| No - id: 117b47a4-0b41-4d0e-8dc1-2c2eb76befda - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' + name: grammatical_between_A_B reference: '' - 27988d5f-9575-460e-a572-5dc150e2ac32: !Template + 6a6296d1-b066-45f2-8550-915f8cc3a4be: !Template answer_choices: Yes ||| No - id: 27988d5f-9575-460e-a572-5dc150e2ac32 + id: 6a6296d1-b066-45f2-8550-915f8cc3a4be jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' + name: single_sentence_good_no_yes reference: '' - 27d930bd-417f-4468-a2f1-3e700832c3ba: !Template - answer_choices: null - id: 27d930bd-417f-4468-a2f1-3e700832c3ba + a79af759-f93a-4f7f-90a9-cf5391a7ca1a: !Template + answer_choices: 1 ||| 2 + id: a79af759-f93a-4f7f-90a9-cf5391a7ca1a jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -136,11 +136,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -148,11 +148,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: grammatical_between_1_2 reference: '' - 350e7dfd-e545-418b-bb5d-70a4a6c7a310: !Template + afbb26fa-da68-4d2d-8333-63f7cadf9586: !Template answer_choices: A ||| B - id: 350e7dfd-e545-418b-bb5d-70a4a6c7a310 + id: afbb26fa-da68-4d2d-8333-63f7cadf9586 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -186,14 +186,30 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: B-A)' + name: better_B_A + reference: '' + c0bdc602-91d0-40e9-af1e-d37b9b66d42f: !Template + answer_choices: Yes ||| No + id: c0bdc602-91d0-40e9-af1e-d37b9b66d42f + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes reference: '' - 3b4b4b58-052c-4505-b7e7-a2e17c17506a: !Template - answer_choices: null - id: 3b4b4b58-052c-4505-b7e7-a2e17c17506a + c4d081c0-7c66-48df-aa56-bc9cf089be76: !Template + answer_choices: A ||| B + id: c4d081c0-7c66-48df-aa56-bc9cf089be76 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -213,11 +229,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -225,15 +241,15 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: grammatical_which_one_A_B reference: '' - 5a3b706b-62d8-4ea4-9a41-da7a60b4d231: !Template + d84b674a-d72c-4bd0-8005-f2f3b4aed72f: !Template answer_choices: Yes ||| No - id: 5a3b706b-62d8-4ea4-9a41-da7a60b4d231 + id: d84b674a-d72c-4bd0-8005-f2f3b4aed72f jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -241,11 +257,11 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' + name: single_sentence_good_yes_no reference: '' - a2f7052f-4615-4323-a468-210db95f3951: !Template + e18be316-c04c-4fa2-81b1-dba8e4c0305b: !Template answer_choices: A ||| B - id: a2f7052f-4615-4323-a468-210db95f3951 + id: e18be316-c04c-4fa2-81b1-dba8e4c0305b jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -279,21 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: A-B)' - reference: '' - f94a694a-ee62-412b-ad6f-071ed15e1204: !Template - answer_choices: Yes ||| No - id: f94a694a-ee62-412b-ad6f-071ed15e1204 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' + name: better_A_B reference: '' diff --git a/promptsource/templates/blimp/ellipsis_n_bar_1/templates.yaml b/promptsource/templates/blimp/ellipsis_n_bar_1/templates.yaml index 0aad747fe..c97d6cfc0 100644 --- a/promptsource/templates/blimp/ellipsis_n_bar_1/templates.yaml +++ b/promptsource/templates/blimp/ellipsis_n_bar_1/templates.yaml @@ -1,40 +1,24 @@ dataset: blimp subset: ellipsis_n_bar_1 templates: - 3752e43c-6907-4cb0-a5be-251704c4227d: !Template - answer_choices: Yes ||| No - id: 3752e43c-6907-4cb0-a5be-251704c4227d - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' - reference: '' - 522aa603-fad1-4272-84f2-8069a4d5f0f5: !Template - answer_choices: null - id: 522aa603-fad1-4272-84f2-8069a4d5f0f5 + 061cf5f0-d23f-47b4-b481-bc3bf9d9abe6: !Template + answer_choices: 1 ||| 2 + id: 061cf5f0-d23f-47b4-b481-bc3bf9d9abe6 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -42,11 +26,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -54,14 +38,14 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: grammatical_which_one_1_2 reference: '' - 7655f660-5815-4e41-9f55-a3a3400ff80c: !Template - answer_choices: null - id: 7655f660-5815-4e41-9f55-a3a3400ff80c + 0629359c-ec2a-4853-b7d2-87cdfb65645d: !Template + answer_choices: 1 ||| 2 + id: 0629359c-ec2a-4853-b7d2-87cdfb65645d jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -81,11 +65,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -93,15 +77,69 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: grammatical_between_1_2 reference: '' - 964bcb7a-5364-44ec-8fa8-eb21308387c6: !Template + 253d1bc9-dec2-4173-a047-6714ee4716b0: !Template answer_choices: Yes ||| No - id: 964bcb7a-5364-44ec-8fa8-eb21308387c6 + id: 253d1bc9-dec2-4173-a047-6714ee4716b0 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes + reference: '' + 34909b4a-ad88-4899-a05a-9e2465a790f8: !Template + answer_choices: A ||| B + id: 34909b4a-ad88-4899-a05a-9e2465a790f8 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: better_B_A + reference: '' + 3ea2c661-b45e-42a1-b962-ca9cef661ae1: !Template + answer_choices: Yes ||| No + id: 3ea2c661-b45e-42a1-b962-ca9cef661ae1 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -109,27 +147,27 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' + name: single_sentence_good_no_yes reference: '' - 97bfbcd8-13a8-4be9-b2d6-32a2d1c9fecf: !Template + 681bb1fd-a79e-403d-830d-1a138293741e: !Template answer_choices: Yes ||| No - id: 97bfbcd8-13a8-4be9-b2d6-32a2d1c9fecf + id: 681bb1fd-a79e-403d-830d-1a138293741e jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' + name: single_sentence_good_yes_no reference: '' - b55558f4-73c9-4d70-a80c-b16e8c459963: !Template - answer_choices: null - id: b55558f4-73c9-4d70-a80c-b16e8c459963 + 8e1bcbb0-ff9a-4b00-a325-dbb758681043: !Template + answer_choices: A ||| B + id: 8e1bcbb0-ff9a-4b00-a325-dbb758681043 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer A or B. @@ -152,11 +190,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -164,11 +202,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: grammatical_which_one_A_B reference: '' - bbb01915-634c-4728-8a06-573cab8996f5: !Template + a45b964c-c296-46b7-87ec-4a3d7d8f9098: !Template answer_choices: A ||| B - id: bbb01915-634c-4728-8a06-573cab8996f5 + id: a45b964c-c296-46b7-87ec-4a3d7d8f9098 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -187,7 +225,7 @@ templates: {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order + Which one is a better sentence of English, A or B? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -202,42 +240,42 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: B-A)' + name: better_A_B reference: '' - d7c98f47-a090-4462-bc92-59994ab98636: !Template + c4ee8fa5-2b8a-44de-a630-9794cc6dcc51: !Template answer_choices: Yes ||| No - id: d7c98f47-a090-4462-bc92-59994ab98636 + id: c4ee8fa5-2b8a-44de-a630-9794cc6dcc51 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' + name: single_sentence_bad_yes_no reference: '' - dd32551a-04e6-43e9-b6ce-0f28343ad36b: !Template - answer_choices: null - id: dd32551a-04e6-43e9-b6ce-0f28343ad36b + de9f89ef-d20d-45c9-a22c-3f229a465094: !Template + answer_choices: A ||| B + id: de9f89ef-d20d-45c9-a22c-3f229a465094 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -245,44 +283,6 @@ templates: {% if shuffled_order == 0 %} - 1 - - {% else %} - - 2 - - {% endif %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: random_choice_1_2_2 - reference: '' - f0d98fcc-bb1c-47ad-8501-8059becb674d: !Template - answer_choices: A ||| B - id: f0d98fcc-bb1c-47ad-8501-8059becb674d - jinja: '{% set shuffled_order = [0, 1] | random %} - - Read the following two sentences. - - {% if shuffled_order == 0 %} - - Sentence A: {{ sentence_good }} - - Sentence B: {{ sentence_bad }} - - {% else %} - - Sentence A: {{ sentence_bad }} - - Sentence B: {{ sentence_good }} - - {% endif %} - - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} - {{ answer_choices[0] }} {% else %} @@ -295,5 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: A-B)' + name: grammatical_between_A_B reference: '' diff --git a/promptsource/templates/blimp/ellipsis_n_bar_2/templates.yaml b/promptsource/templates/blimp/ellipsis_n_bar_2/templates.yaml index ad9aa33c2..297b5d886 100644 --- a/promptsource/templates/blimp/ellipsis_n_bar_2/templates.yaml +++ b/promptsource/templates/blimp/ellipsis_n_bar_2/templates.yaml @@ -1,40 +1,40 @@ dataset: blimp subset: ellipsis_n_bar_2 templates: - 0b780cde-1f1f-4dba-8a71-b7668d7da967: !Template + 02cf9b03-67fc-414a-b3f1-bcdd0f65929b: !Template answer_choices: Yes ||| No - id: 0b780cde-1f1f-4dba-8a71-b7668d7da967 + id: 02cf9b03-67fc-414a-b3f1-bcdd0f65929b jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' + name: single_sentence_bad_yes_no reference: '' - 1e37fcb2-4f57-4fd2-b529-617b0a43b09c: !Template - answer_choices: null - id: 1e37fcb2-4f57-4fd2-b529-617b0a43b09c + 08c26355-4c8b-4bf6-95b0-16025b65bd25: !Template + answer_choices: A ||| B + id: 08c26355-4c8b-4bf6-95b0-16025b65bd25 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -42,44 +42,6 @@ templates: {% if shuffled_order == 0 %} - 1 - - {% else %} - - 2 - - {% endif %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: random_choice_1_2_2 - reference: '' - 378f0de8-ea2b-40c0-b33d-16ad3b1bf1d9: !Template - answer_choices: A ||| B - id: 378f0de8-ea2b-40c0-b33d-16ad3b1bf1d9 - jinja: '{% set shuffled_order = [0, 1] | random %} - - Read the following two sentences. - - {% if shuffled_order == 0 %} - - Sentence A: {{ sentence_good }} - - Sentence B: {{ sentence_bad }} - - {% else %} - - Sentence A: {{ sentence_bad }} - - Sentence B: {{ sentence_good }} - - {% endif %} - - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} - {{ answer_choices[0] }} {% else %} @@ -92,15 +54,15 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: B-A)' + name: grammatical_which_one_A_B reference: '' - 80371c00-36d5-4c85-935a-cccbb98aea43: !Template + 25e742f5-a100-4f06-a1eb-a9203afa8427: !Template answer_choices: Yes ||| No - id: 80371c00-36d5-4c85-935a-cccbb98aea43 + id: 25e742f5-a100-4f06-a1eb-a9203afa8427 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -108,11 +70,11 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' + name: single_sentence_good_yes_no reference: '' - 89972adf-1163-408e-9412-cf89c8845282: !Template - answer_choices: null - id: 89972adf-1163-408e-9412-cf89c8845282 + 2f68abe6-8abc-4e7e-ad3f-af512ec8e7b6: !Template + answer_choices: 1 ||| 2 + id: 2f68abe6-8abc-4e7e-ad3f-af512ec8e7b6 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer 1 or 2. @@ -135,11 +97,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -147,14 +109,30 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: grammatical_which_one_1_2 reference: '' - b9f576ab-362d-4876-9f0a-02d9392ff2f6: !Template - answer_choices: null - id: b9f576ab-362d-4876-9f0a-02d9392ff2f6 + 4245a98b-a715-47b1-ac54-9f8eb344840b: !Template + answer_choices: Yes ||| No + id: 4245a98b-a715-47b1-ac54-9f8eb344840b + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes + reference: '' + 5a8b5b95-b762-41a4-8367-6e28f4f1eeef: !Template + answer_choices: A ||| B + id: 5a8b5b95-b762-41a4-8367-6e28f4f1eeef jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -174,11 +152,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -186,27 +164,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B - reference: '' - bd6b8408-b8e4-4f6d-9b67-74062d0cb696: !Template - answer_choices: Yes ||| No - id: bd6b8408-b8e4-4f6d-9b67-74062d0cb696 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' + name: grammatical_between_A_B reference: '' - c492684e-aa70-488a-9098-2f06ef52cf1d: !Template + 5d6c9179-d290-40f4-b0e9-9446182f14cb: !Template answer_choices: A ||| B - id: c492684e-aa70-488a-9098-2f06ef52cf1d + id: 5d6c9179-d290-40f4-b0e9-9446182f14cb jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -240,54 +202,76 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: A-B)' + name: better_A_B reference: '' - d58eae09-f150-43a2-9f81-17c25ec82d30: !Template - answer_choices: Yes ||| No - id: d58eae09-f150-43a2-9f81-17c25ec82d30 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + 67f471fd-ebf0-44b2-92e5-4775a16cfd5a: !Template + answer_choices: 1 ||| 2 + id: 67f471fd-ebf0-44b2-92e5-4775a16cfd5a + jinja: '{% set shuffled_order = [0, 1] | random %} - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' + original_task: true + name: grammatical_between_1_2 reference: '' - dbc37147-56c2-4898-b632-66db234e6d21: !Template - answer_choices: null - id: dbc37147-56c2-4898-b632-66db234e6d21 + a6ce1219-883b-4af2-ab8c-223adca927d9: !Template + answer_choices: A ||| B + id: a6ce1219-883b-4af2-ab8c-223adca927d9 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -295,5 +279,21 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: better_B_A + reference: '' + e51c0a4e-f7bc-4f10-b3c4-161b5e8abd32: !Template + answer_choices: Yes ||| No + id: e51c0a4e-f7bc-4f10-b3c4-161b5e8abd32 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes reference: '' diff --git a/promptsource/templates/blimp/existential_there_object_raising/templates.yaml b/promptsource/templates/blimp/existential_there_object_raising/templates.yaml index 5f28f6548..57f65c390 100644 --- a/promptsource/templates/blimp/existential_there_object_raising/templates.yaml +++ b/promptsource/templates/blimp/existential_there_object_raising/templates.yaml @@ -1,13 +1,13 @@ dataset: blimp subset: existential_there_object_raising templates: - 28a55ee2-d8f6-4fc3-8008-e60150559377: !Template + 1181c64d-a4c9-48ee-a3d7-a0387d1716e1: !Template answer_choices: Yes ||| No - id: 28a55ee2-d8f6-4fc3-8008-e60150559377 + id: 1181c64d-a4c9-48ee-a3d7-a0387d1716e1 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata @@ -15,38 +15,37 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' + name: single_sentence_bad_yes_no reference: '' - 6921d555-378b-4629-8d75-a11bad991c5a: !Template - answer_choices: null - id: 6921d555-378b-4629-8d75-a11bad991c5a + 25cf67cc-c980-4e7a-9ef0-1f860f2813b8: !Template + answer_choices: A ||| B + id: 25cf67cc-c980-4e7a-9ef0-1f860f2813b8 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -54,47 +53,32 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 - reference: '' - 776cdf8b-ee71-4b59-9873-a0d3a07ae77d: !Template - answer_choices: Yes ||| No - id: 776cdf8b-ee71-4b59-9873-a0d3a07ae77d - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' + name: better_A_B reference: '' - 8edb1a66-2e13-47f4-92c5-804ce8e947a2: !Template - answer_choices: A ||| B - id: 8edb1a66-2e13-47f4-92c5-804ce8e947a2 + 278cc4e0-1c44-4786-828b-52858e29ee66: !Template + answer_choices: 1 ||| 2 + id: 278cc4e0-1c44-4786-828b-52858e29ee66 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -108,30 +92,14 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: B-A)' - reference: '' - a0cad0f5-22f7-43ae-a3ab-b52c8b842d28: !Template - answer_choices: Yes ||| No - id: a0cad0f5-22f7-43ae-a3ab-b52c8b842d28 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' + name: grammatical_which_one_1_2 reference: '' - be07094b-ed75-4118-b2da-024077dedd26: !Template - answer_choices: null - id: be07094b-ed75-4118-b2da-024077dedd26 + 2f909ebc-9ded-4fb3-818a-6be2c000e9d6: !Template + answer_choices: A ||| B + id: 2f909ebc-9ded-4fb3-818a-6be2c000e9d6 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -151,11 +119,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -163,15 +131,15 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: grammatical_which_one_A_B reference: '' - cb9a8767-7450-4074-8037-aa054f24db81: !Template + 3f62cc54-edab-4d7b-93ee-6f716464c5d3: !Template answer_choices: Yes ||| No - id: cb9a8767-7450-4074-8037-aa054f24db81 + id: 3f62cc54-edab-4d7b-93ee-6f716464c5d3 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata @@ -179,14 +147,14 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' + name: single_sentence_bad_no_yes reference: '' - e21618f9-8973-4236-91f6-c88ad17fe846: !Template - answer_choices: null - id: e21618f9-8973-4236-91f6-c88ad17fe846 + 6f37b12f-0176-46a4-93b5-4432b3a8ccb2: !Template + answer_choices: A ||| B + id: 6f37b12f-0176-46a4-93b5-4432b3a8ccb2 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -206,11 +174,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -218,11 +186,27 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: grammatical_between_A_B + reference: '' + 7a8e4d4d-2b2a-4890-bb57-34c33ee66e15: !Template + answer_choices: Yes ||| No + id: 7a8e4d4d-2b2a-4890-bb57-34c33ee66e15 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no reference: '' - e72a0571-fbee-47d5-bad0-18c039550f99: !Template - answer_choices: null - id: e72a0571-fbee-47d5-bad0-18c039550f99 + 95c3afa9-20b9-479f-bf26-9d4f46c03b1b: !Template + answer_choices: 1 ||| 2 + id: 95c3afa9-20b9-479f-bf26-9d4f46c03b1b jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -245,11 +229,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -257,11 +241,27 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: grammatical_between_1_2 + reference: '' + a64d94c9-9b06-4a4c-83bd-c07f49969c9e: !Template + answer_choices: Yes ||| No + id: a64d94c9-9b06-4a4c-83bd-c07f49969c9e + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes reference: '' - fc31a373-2721-4623-a0ae-f91eb8c2ef36: !Template + dbcf12cd-3dad-499e-9f67-bb9cf496842f: !Template answer_choices: A ||| B - id: fc31a373-2721-4623-a0ae-f91eb8c2ef36 + id: dbcf12cd-3dad-499e-9f67-bb9cf496842f jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -280,7 +280,7 @@ templates: {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order + Which one is a better sentence of English, B or A? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -295,5 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: A-B)' + name: better_B_A reference: '' diff --git a/promptsource/templates/blimp/existential_there_quantifiers_1/templates.yaml b/promptsource/templates/blimp/existential_there_quantifiers_1/templates.yaml index 134921c89..b2ee9a440 100644 --- a/promptsource/templates/blimp/existential_there_quantifiers_1/templates.yaml +++ b/promptsource/templates/blimp/existential_there_quantifiers_1/templates.yaml @@ -1,12 +1,12 @@ dataset: blimp subset: existential_there_quantifiers_1 templates: - 0233dd43-9281-4398-a134-1083fde228aa: !Template - answer_choices: null - id: 0233dd43-9281-4398-a134-1083fde228aa + 0ed94cec-4972-4d68-a41c-13cd3ae69a0b: !Template + answer_choices: A ||| B + id: 0ed94cec-4972-4d68-a41c-13cd3ae69a0b jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -26,11 +26,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -38,14 +38,14 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: grammatical_which_one_A_B reference: '' - 0e144c21-4ab8-40d1-8ac5-32a4bf0c8f49: !Template - answer_choices: null - id: 0e144c21-4ab8-40d1-8ac5-32a4bf0c8f49 + 374e8fdf-e845-4858-80c6-91de64c48954: !Template + answer_choices: 1 ||| 2 + id: 374e8fdf-e845-4858-80c6-91de64c48954 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -65,11 +65,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -77,11 +77,27 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: grammatical_between_1_2 reference: '' - 3b60d1d4-eccf-46be-a1c3-2d848175ad62: !Template + 3a86efed-1f19-4236-9623-f7c9913ff79a: !Template + answer_choices: Yes ||| No + id: 3a86efed-1f19-4236-9623-f7c9913ff79a + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 50a44f93-2e19-4e1e-8a16-f421ca6f8d36: !Template answer_choices: A ||| B - id: 3b60d1d4-eccf-46be-a1c3-2d848175ad62 + id: 50a44f93-2e19-4e1e-8a16-f421ca6f8d36 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -115,26 +131,26 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: B-A)' + name: better_B_A reference: '' - 51ebc961-3d22-4723-bb9b-39a930c43615: !Template - answer_choices: null - id: 51ebc961-3d22-4723-bb9b-39a930c43615 + 762f5547-0b18-4b1c-a66c-a5b2c1dbdb07: !Template + answer_choices: 1 ||| 2 + id: 762f5547-0b18-4b1c-a66c-a5b2c1dbdb07 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -142,11 +158,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -154,42 +170,26 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B - reference: '' - 55469408-4881-4c2a-a4b7-a0c23d31c52f: !Template - answer_choices: Yes ||| No - id: 55469408-4881-4c2a-a4b7-a0c23d31c52f - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' + name: grammatical_which_one_1_2 reference: '' - 559cca5d-4e00-400b-a161-ced8cd373d5d: !Template - answer_choices: null - id: 559cca5d-4e00-400b-a161-ced8cd373d5d + 94ddff00-ce06-41a3-86df-044b57b9c989: !Template + answer_choices: A ||| B + id: 94ddff00-ce06-41a3-86df-044b57b9c989 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -197,11 +197,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -209,11 +209,27 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: grammatical_between_A_B + reference: '' + ceaa3f86-04e5-4e53-a007-1a40994629f0: !Template + answer_choices: Yes ||| No + id: ceaa3f86-04e5-4e53-a007-1a40994629f0 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no reference: '' - 78895df4-c60c-4be2-ba06-eb0f5ddbdb4a: !Template + d39ea036-c626-4487-88be-48a168a99726: !Template answer_choices: A ||| B - id: 78895df4-c60c-4be2-ba06-eb0f5ddbdb4a + id: d39ea036-c626-4487-88be-48a168a99726 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -247,31 +263,15 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: A-B)' + name: better_A_B reference: '' - 79d8ce45-cbe4-486a-ab84-7426e4f30790: !Template + e8239a92-911a-4a48-b5da-60507c0e6081: !Template answer_choices: Yes ||| No - id: 79d8ce45-cbe4-486a-ab84-7426e4f30790 + id: e8239a92-911a-4a48-b5da-60507c0e6081 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' - reference: '' - 9e82e459-6af7-4c62-b382-eeb7c88ed5b3: !Template - answer_choices: Yes ||| No - id: 9e82e459-6af7-4c62-b382-eeb7c88ed5b3 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata @@ -279,21 +279,21 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' + name: single_sentence_bad_no_yes reference: '' - c93a3c8e-35ca-4a86-bcb2-dc9c19ee423e: !Template + fac46819-b60e-4694-be22-3074d3d914a0: !Template answer_choices: Yes ||| No - id: c93a3c8e-35ca-4a86-bcb2-dc9c19ee423e + id: fac46819-b60e-4694-be22-3074d3d914a0 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' + name: single_sentence_good_no_yes reference: '' diff --git a/promptsource/templates/blimp/existential_there_quantifiers_2/templates.yaml b/promptsource/templates/blimp/existential_there_quantifiers_2/templates.yaml index 546ecc206..e5c5486f9 100644 --- a/promptsource/templates/blimp/existential_there_quantifiers_2/templates.yaml +++ b/promptsource/templates/blimp/existential_there_quantifiers_2/templates.yaml @@ -1,48 +1,25 @@ dataset: blimp subset: existential_there_quantifiers_2 templates: - 228f312c-752d-4813-87d4-d20f50bf1e65: !Template - answer_choices: null - id: 228f312c-752d-4813-87d4-d20f50bf1e65 - jinja: '{% set shuffled_order = [0, 1] | random %} - - Between the two sentences, which one is grammatical? Please answer A or B. - - {% if shuffled_order == 0 %} - - A: {{ sentence_good }} - - B: {{ sentence_bad }} - - {% else %} - - A: {{ sentence_bad }} - - B: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - A - - {% else %} - - B + 1f21002e-fb16-4aa7-82c1-449dc8c2da47: !Template + answer_choices: Yes ||| No + id: 1f21002e-fb16-4aa7-82c1-449dc8c2da47 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - {% endif %}' + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: random_choice_A_B_2 + original_task: false + name: single_sentence_good_yes_no reference: '' - 3c1d7194-b4da-4123-9546-e7a94a1c6a07: !Template + 20661c62-c2e1-4543-8248-4a4674448cbc: !Template answer_choices: Yes ||| No - id: 3c1d7194-b4da-4123-9546-e7a94a1c6a07 + id: 20661c62-c2e1-4543-8248-4a4674448cbc jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -54,38 +31,37 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' + name: single_sentence_good_no_yes reference: '' - 5ea32d10-921f-4358-b1de-f93ee3a325f5: !Template - answer_choices: null - id: 5ea32d10-921f-4358-b1de-f93ee3a325f5 + 342eb8f1-a729-4834-aae1-ccf6972f0529: !Template + answer_choices: A ||| B + id: 342eb8f1-a729-4834-aae1-ccf6972f0529 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -93,31 +69,32 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: better_B_A reference: '' - 629052b1-e99b-490f-ace0-a369011de80e: !Template + 6cc4b292-85f5-43b5-acc1-d84e8345ae5e: !Template answer_choices: A ||| B - id: 629052b1-e99b-490f-ace0-a369011de80e + id: 6cc4b292-85f5-43b5-acc1-d84e8345ae5e jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -131,58 +108,81 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: B-A)' + name: grammatical_between_A_B reference: '' - 69b7033a-0bfb-4412-b846-401710062776: !Template + 86521763-1a17-4c88-a6d7-5b9e5b6bba2c: !Template answer_choices: Yes ||| No - id: 69b7033a-0bfb-4412-b846-401710062776 + id: 86521763-1a17-4c88-a6d7-5b9e5b6bba2c jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' + name: single_sentence_bad_yes_no reference: '' - 8c52b91f-2bd4-443e-b721-2b539b6b153c: !Template - answer_choices: Yes ||| No - id: 8c52b91f-2bd4-443e-b721-2b539b6b153c - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + 8b069711-571b-4b45-b050-7ea4b0a3d472: !Template + answer_choices: 1 ||| 2 + id: 8b069711-571b-4b45-b050-7ea4b0a3d472 + jinja: '{% set shuffled_order = [0, 1] | random %} - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' + original_task: true + name: grammatical_between_1_2 reference: '' - ca69fdaa-db5b-4cb6-8d3e-2db4b5933e18: !Template - answer_choices: null - id: ca69fdaa-db5b-4cb6-8d3e-2db4b5933e18 + ad7120e1-87f2-4144-a900-ba99b07364c5: !Template + answer_choices: 1 ||| 2 + id: ad7120e1-87f2-4144-a900-ba99b07364c5 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -190,11 +190,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -202,15 +202,15 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: grammatical_which_one_1_2 reference: '' - dc29e12a-b8fa-4791-a90e-faaaca8471c4: !Template + b1afcf7c-0086-41bb-a892-627aeedd27dc: !Template answer_choices: Yes ||| No - id: dc29e12a-b8fa-4791-a90e-faaaca8471c4 + id: b1afcf7c-0086-41bb-a892-627aeedd27dc jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata @@ -218,38 +218,37 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' + name: single_sentence_bad_no_yes reference: '' - ddb8419e-c454-4a03-8fa3-853d65aa3565: !Template - answer_choices: null - id: ddb8419e-c454-4a03-8fa3-853d65aa3565 + c5cf39d6-a801-4ec8-840d-f569af4786b5: !Template + answer_choices: A ||| B + id: c5cf39d6-a801-4ec8-840d-f569af4786b5 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -257,31 +256,32 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: better_A_B reference: '' - e424dc78-4dc2-429d-ade4-5e18a2cd41b1: !Template + e8338a6f-2fd3-4aac-8891-b70553f7eb04: !Template answer_choices: A ||| B - id: e424dc78-4dc2-429d-ade4-5e18a2cd41b1 + id: e8338a6f-2fd3-4aac-8891-b70553f7eb04 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -295,5 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: A-B)' + name: grammatical_which_one_A_B reference: '' diff --git a/promptsource/templates/blimp/existential_there_subject_raising/templates.yaml b/promptsource/templates/blimp/existential_there_subject_raising/templates.yaml index 8d14628d0..18485c6f5 100644 --- a/promptsource/templates/blimp/existential_there_subject_raising/templates.yaml +++ b/promptsource/templates/blimp/existential_there_subject_raising/templates.yaml @@ -1,25 +1,47 @@ dataset: blimp subset: existential_there_subject_raising templates: - 00796a33-5b42-4651-b901-36e08ab9dea6: !Template - answer_choices: Yes ||| No - id: 00796a33-5b42-4651-b901-36e08ab9dea6 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + 370bd524-e7f2-4fe7-8191-db3b32cc6e20: !Template + answer_choices: A ||| B + id: 370bd524-e7f2-4fe7-8191-db3b32cc6e20 + jinja: '{% set shuffled_order = [0, 1] | random %} - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' + original_task: true + name: better_A_B reference: '' - 16b36ab4-e933-433c-9c9e-19147c95bebe: !Template - answer_choices: null - id: 16b36ab4-e933-433c-9c9e-19147c95bebe + 3fbce7f4-5596-4ed1-9aec-e23e85738cfa: !Template + answer_choices: A ||| B + id: 3fbce7f4-5596-4ed1-9aec-e23e85738cfa jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer A or B. @@ -42,11 +64,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -54,31 +76,48 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: grammatical_between_A_B reference: '' - 61b49b7b-e775-4ad4-ae18-d19f89bb792f: !Template + 59581add-819b-496b-8d61-cf373f163567: !Template + answer_choices: Yes ||| No + id: 59581add-819b-496b-8d61-cf373f163567 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes + reference: '' + 71e38631-705c-47b8-a836-b8ab2e9c2329: !Template answer_choices: A ||| B - id: 61b49b7b-e775-4ad4-ae18-d19f89bb792f + id: 71e38631-705c-47b8-a836-b8ab2e9c2329 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -92,47 +131,48 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: B-A)' + name: grammatical_which_one_A_B reference: '' - 81129215-698a-449d-a299-3f38c06fb0ba: !Template + 7bb5c9ac-5c34-4c72-a273-981c13d03ef8: !Template answer_choices: Yes ||| No - id: 81129215-698a-449d-a299-3f38c06fb0ba + id: 7bb5c9ac-5c34-4c72-a273-981c13d03ef8 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' + name: single_sentence_bad_yes_no reference: '' - 875fac89-3cd6-46ce-b544-71e5bf2bde4f: !Template - answer_choices: A ||| B - id: 875fac89-3cd6-46ce-b544-71e5bf2bde4f + 84a0b0ee-a021-4feb-833a-c982c6e55150: !Template + answer_choices: 1 ||| 2 + id: 84a0b0ee-a021-4feb-833a-c982c6e55150 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -146,26 +186,26 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: A-B)' + name: grammatical_between_1_2 reference: '' - 8f9b0b81-cc49-43c5-918e-afd2df6a0914: !Template - answer_choices: null - id: 8f9b0b81-cc49-43c5-918e-afd2df6a0914 + 88ecc6bc-9887-410a-bf59-426ebf0a9d52: !Template + answer_choices: 1 ||| 2 + id: 88ecc6bc-9887-410a-bf59-426ebf0a9d52 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -173,11 +213,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -185,38 +225,37 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: grammatical_which_one_1_2 reference: '' - c753e480-c114-4dee-9b03-c5351b09a47f: !Template - answer_choices: null - id: c753e480-c114-4dee-9b03-c5351b09a47f + 8ec321ed-96f6-4c86-83b2-e2b6aa3688af: !Template + answer_choices: A ||| B + id: 8ec321ed-96f6-4c86-83b2-e2b6aa3688af jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -224,31 +263,31 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: better_B_A reference: '' - d6f43663-86b6-44ed-976f-95added0fb4e: !Template + a43ac65c-c607-428d-8603-45154cff7d93: !Template answer_choices: Yes ||| No - id: d6f43663-86b6-44ed-976f-95added0fb4e + id: a43ac65c-c607-428d-8603-45154cff7d93 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' + name: single_sentence_good_yes_no reference: '' - e5f3b1c7-ac29-4791-9eb6-aa98612b8f9a: !Template + ae4a0b4a-dc6f-473c-9fb9-9c9d0a35f722: !Template answer_choices: Yes ||| No - id: e5f3b1c7-ac29-4791-9eb6-aa98612b8f9a + id: ae4a0b4a-dc6f-473c-9fb9-9c9d0a35f722 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -256,44 +295,5 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' - reference: '' - e75c847d-96b2-4618-b616-fbbe88a2af0e: !Template - answer_choices: null - id: e75c847d-96b2-4618-b616-fbbe88a2af0e - jinja: '{% set shuffled_order = [0, 1] | random %} - - Between the two sentences, which one is grammatical? Please answer 1 or 2. - - {% if shuffled_order == 0 %} - - 1: {{ sentence_good }} - - 2: {{ sentence_bad }} - - {% else %} - - 1: {{ sentence_bad }} - - 2: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - 1 - - {% else %} - - 2 - - {% endif %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: random_choice_1_2_2 + name: single_sentence_good_no_yes reference: '' diff --git a/promptsource/templates/blimp/expletive_it_object_raising/templates.yaml b/promptsource/templates/blimp/expletive_it_object_raising/templates.yaml index 06e8eaac5..f342637bc 100644 --- a/promptsource/templates/blimp/expletive_it_object_raising/templates.yaml +++ b/promptsource/templates/blimp/expletive_it_object_raising/templates.yaml @@ -1,56 +1,24 @@ dataset: blimp subset: expletive_it_object_raising templates: - 07225df9-d8a8-4715-b7e0-9a38fa5f527b: !Template - answer_choices: Yes ||| No - id: 07225df9-d8a8-4715-b7e0-9a38fa5f527b - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' - reference: '' - 339a7a6d-d2c8-45c9-bb1c-a089a084f099: !Template - answer_choices: Yes ||| No - id: 339a7a6d-d2c8-45c9-bb1c-a089a084f099 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' - reference: '' - 341b5119-123a-4101-b75e-ab0fee57f534: !Template - answer_choices: null - id: 341b5119-123a-4101-b75e-ab0fee57f534 + 0401f307-fd12-4067-9f1d-ba3cbe169835: !Template + answer_choices: A ||| B + id: 0401f307-fd12-4067-9f1d-ba3cbe169835 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -58,11 +26,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -70,31 +38,32 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: grammatical_between_A_B reference: '' - 58d8a96f-1c6a-48e4-8545-716f60211a5e: !Template + 05627b8a-cb03-4ee6-9a5d-8580468b6750: !Template answer_choices: A ||| B - id: 58d8a96f-1c6a-48e4-8545-716f60211a5e + id: 05627b8a-cb03-4ee6-9a5d-8580468b6750 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -108,43 +77,11 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: A-B)' - reference: '' - 83410312-22b9-4ff2-a3af-387752a104e6: !Template - answer_choices: Yes ||| No - id: 83410312-22b9-4ff2-a3af-387752a104e6 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' + name: grammatical_which_one_A_B reference: '' - 853fd6b5-7bf4-40bf-99f2-8401cff9d5c1: !Template - answer_choices: Yes ||| No - id: 853fd6b5-7bf4-40bf-99f2-8401cff9d5c1 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' - reference: '' - 9bd3eea0-08e6-42d0-8088-6589e67d5578: !Template + 07d86f3e-935a-42f7-bf65-efb6e528ff7e: !Template answer_choices: A ||| B - id: 9bd3eea0-08e6-42d0-8088-6589e67d5578 + id: 07d86f3e-935a-42f7-bf65-efb6e528ff7e jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -163,7 +100,7 @@ templates: {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order + Which one is a better sentence of English, A or B? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -178,26 +115,42 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: B-A)' + name: better_A_B + reference: '' + 10be0bc5-5081-4c9e-8e05-f6d291d72aed: !Template + answer_choices: Yes ||| No + id: 10be0bc5-5081-4c9e-8e05-f6d291d72aed + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes reference: '' - b91f51aa-e97e-4d13-86e5-d965aeb8783a: !Template - answer_choices: null - id: b91f51aa-e97e-4d13-86e5-d965aeb8783a + 190d288d-a1f0-4013-b89d-cd7024eac333: !Template + answer_choices: 1 ||| 2 + id: 190d288d-a1f0-4013-b89d-cd7024eac333 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -205,11 +158,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -217,14 +170,30 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: grammatical_which_one_1_2 reference: '' - eafec0bf-8db2-47d4-bc13-4ec30d2b1d37: !Template - answer_choices: null - id: eafec0bf-8db2-47d4-bc13-4ec30d2b1d37 + 22537da5-d41b-4a0a-ba6b-5a169615ebca: !Template + answer_choices: Yes ||| No + id: 22537da5-d41b-4a0a-ba6b-5a169615ebca + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 36ada838-12df-4a90-8129-c163a16f23b8: !Template + answer_choices: 1 ||| 2 + id: 36ada838-12df-4a90-8129-c163a16f23b8 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -244,11 +213,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -256,38 +225,53 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: grammatical_between_1_2 + reference: '' + b9c464ed-634c-426e-bbd3-43b99a790110: !Template + answer_choices: Yes ||| No + id: b9c464ed-634c-426e-bbd3-43b99a790110 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no reference: '' - fc121e5a-867e-4c74-8f2b-9419a8a8c984: !Template - answer_choices: null - id: fc121e5a-867e-4c74-8f2b-9419a8a8c984 + f0de1f54-d371-4e1b-af4e-ea9c83cf0226: !Template + answer_choices: A ||| B + id: f0de1f54-d371-4e1b-af4e-ea9c83cf0226 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -295,5 +279,21 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: better_B_A + reference: '' + f85dad10-90f2-427a-887a-d2fb243a54b5: !Template + answer_choices: Yes ||| No + id: f85dad10-90f2-427a-887a-d2fb243a54b5 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes reference: '' diff --git a/promptsource/templates/blimp/inchoative/templates.yaml b/promptsource/templates/blimp/inchoative/templates.yaml index da70a52b1..0aca8bbea 100644 --- a/promptsource/templates/blimp/inchoative/templates.yaml +++ b/promptsource/templates/blimp/inchoative/templates.yaml @@ -1,45 +1,30 @@ dataset: blimp subset: inchoative templates: - 18fd76ff-1ae0-4e76-b89a-87c6821dbc24: !Template - answer_choices: Yes ||| No - id: 18fd76ff-1ae0-4e76-b89a-87c6821dbc24 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' - reference: '' - 4c38c931-44e4-452f-a798-37050ee40bb2: !Template + 35ce6b18-a935-4b84-9665-f0e7e5b28591: !Template answer_choices: A ||| B - id: 4c38c931-44e4-452f-a798-37050ee40bb2 + id: 35ce6b18-a935-4b84-9665-f0e7e5b28591 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -53,30 +38,53 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: B-A)' + name: grammatical_between_A_B reference: '' - 5c5c1067-0b10-4d55-9d8d-ea15a6eba76b: !Template - answer_choices: Yes ||| No - id: 5c5c1067-0b10-4d55-9d8d-ea15a6eba76b - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + 39734e83-f9d4-4fd1-aaff-56e218bf7406: !Template + answer_choices: 1 ||| 2 + id: 39734e83-f9d4-4fd1-aaff-56e218bf7406 + jinja: '{% set shuffled_order = [0, 1] | random %} - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' + original_task: true + name: grammatical_between_1_2 reference: '' - 70cf6acb-c3bf-4fb9-a5fc-6ab341fe8d3f: !Template - answer_choices: null - id: 70cf6acb-c3bf-4fb9-a5fc-6ab341fe8d3f + 7aadbc5d-c690-4ed1-a85a-06634ee52a1c: !Template + answer_choices: A ||| B + id: 7aadbc5d-c690-4ed1-a85a-06634ee52a1c jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -96,11 +104,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -108,15 +116,15 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: grammatical_which_one_A_B reference: '' - 7ea9cc59-e8da-40f5-bd1a-8177df6d7785: !Template + 83d3a865-30f4-443e-8368-7336e58295f9: !Template answer_choices: Yes ||| No - id: 7ea9cc59-e8da-40f5-bd1a-8177df6d7785 + id: 83d3a865-30f4-443e-8368-7336e58295f9 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -124,11 +132,11 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' + name: single_sentence_good_yes_no reference: '' - 9be06dc8-6052-4faf-b6e3-2648ca0c6830: !Template + 8d1b0578-195b-43c8-b4ae-690c36f580f0: !Template answer_choices: A ||| B - id: 9be06dc8-6052-4faf-b6e3-2648ca0c6830 + id: 8d1b0578-195b-43c8-b4ae-690c36f580f0 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -162,54 +170,47 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: A-B)' + name: better_A_B reference: '' - a20e3b81-3480-4899-a9b2-0c795c620dde: !Template - answer_choices: null - id: a20e3b81-3480-4899-a9b2-0c795c620dde - jinja: '{% set shuffled_order = [0, 1] | random %} - - Which one of the following sentences is grammatical? Please answer 1 or 2. - - {% if shuffled_order == 0 %} - - 1: {{ sentence_good }} - - 2: {{ sentence_bad }} - - {% else %} - - 1: {{ sentence_bad }} - - 2: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - 1 - - {% else %} + b1e5958a-c528-4db3-a476-fb946b505800: !Template + answer_choices: Yes ||| No + id: b1e5958a-c528-4db3-a476-fb946b505800 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - 2 + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes + reference: '' + d3ef6e0b-bffe-4967-87b4-f62630976808: !Template + answer_choices: Yes ||| No + id: d3ef6e0b-bffe-4967-87b4-f62630976808 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - {% endif %}' + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: random_choice_1_2 + original_task: false + name: single_sentence_bad_yes_no reference: '' - ac31a784-8c88-4abc-b219-0cae1bfccacf: !Template + d5649722-42bc-4fcb-be3d-3962366290c0: !Template answer_choices: Yes ||| No - id: ac31a784-8c88-4abc-b219-0cae1bfccacf + id: d5649722-42bc-4fcb-be3d-3962366290c0 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -217,14 +218,14 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' + name: single_sentence_good_no_yes reference: '' - cf2c7766-6953-4b83-b5ac-55c20b957b33: !Template - answer_choices: null - id: cf2c7766-6953-4b83-b5ac-55c20b957b33 + f16fb9fb-eb34-4f3c-b5a0-4c90a94fd92e: !Template + answer_choices: 1 ||| 2 + id: f16fb9fb-eb34-4f3c-b5a0-4c90a94fd92e jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -244,11 +245,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -256,38 +257,37 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: grammatical_which_one_1_2 reference: '' - f5efb6e6-b12a-4dc0-8832-503d519afb8e: !Template - answer_choices: null - id: f5efb6e6-b12a-4dc0-8832-503d519afb8e + fc54234e-a248-4bfc-b15c-f179bd434fa1: !Template + answer_choices: A ||| B + id: fc54234e-a248-4bfc-b15c-f179bd434fa1 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -295,5 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: better_B_A reference: '' diff --git a/promptsource/templates/blimp/intransitive/templates.yaml b/promptsource/templates/blimp/intransitive/templates.yaml index c36e3c558..3bacdf798 100644 --- a/promptsource/templates/blimp/intransitive/templates.yaml +++ b/promptsource/templates/blimp/intransitive/templates.yaml @@ -1,9 +1,9 @@ dataset: blimp subset: intransitive templates: - 12123396-d2ac-4b76-9ccf-f7cf87d8dba5: !Template + 0fe12db8-4370-423e-828a-def7a19983d5: !Template answer_choices: A ||| B - id: 12123396-d2ac-4b76-9ccf-f7cf87d8dba5 + id: 0fe12db8-4370-423e-828a-def7a19983d5 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -22,7 +22,7 @@ templates: {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order + Which one is a better sentence of English, A or B? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -37,50 +37,27 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: B-A)' + name: better_A_B reference: '' - 855a5c06-316b-469d-9183-ce64ae57a5db: !Template - answer_choices: null - id: 855a5c06-316b-469d-9183-ce64ae57a5db - jinja: '{% set shuffled_order = [0, 1] | random %} - - Which one of the following sentences is grammatical? Please answer A or B. - - {% if shuffled_order == 0 %} - - A: {{ sentence_good }} - - B: {{ sentence_bad }} - - {% else %} - - A: {{ sentence_bad }} - - B: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - A - - {% else %} - - B + 2f25caac-5beb-444f-aa9b-3d1138e2a5bc: !Template + answer_choices: Yes ||| No + id: 2f25caac-5beb-444f-aa9b-3d1138e2a5bc + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - {% endif %}' + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: random_choice_A_B + original_task: false + name: single_sentence_good_yes_no reference: '' - 87b1169a-409c-491b-9ff9-f5b737fe4e6c: !Template + 4ee3c279-2bd8-45b3-8a62-d09e9566c5a1: !Template answer_choices: Yes ||| No - id: 87b1169a-409c-491b-9ff9-f5b737fe4e6c + id: 4ee3c279-2bd8-45b3-8a62-d09e9566c5a1 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -92,26 +69,26 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' + name: single_sentence_good_no_yes reference: '' - a79100c2-73a8-4b09-bd05-d5974aeff0b6: !Template - answer_choices: null - id: a79100c2-73a8-4b09-bd05-d5974aeff0b6 + 6643bb58-477f-4a75-9673-2617dd2e3772: !Template + answer_choices: 1 ||| 2 + id: 6643bb58-477f-4a75-9673-2617dd2e3772 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -119,11 +96,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -131,31 +108,32 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: grammatical_which_one_1_2 reference: '' - b2cb7019-a27b-4705-8c70-72d9a7d10646: !Template - answer_choices: A ||| B - id: b2cb7019-a27b-4705-8c70-72d9a7d10646 + 6d0fb0f0-07fe-4786-b596-7602ea9e382d: !Template + answer_choices: 1 ||| 2 + id: 6d0fb0f0-07fe-4786-b596-7602ea9e382d jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -169,42 +147,42 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: A-B)' + name: grammatical_between_1_2 reference: '' - b968a4ab-ed23-48cd-9293-2c826072e91b: !Template + 879691af-a2ec-4476-a1c0-0f652093a587: !Template answer_choices: Yes ||| No - id: b968a4ab-ed23-48cd-9293-2c826072e91b + id: 879691af-a2ec-4476-a1c0-0f652093a587 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' + name: single_sentence_bad_yes_no reference: '' - c6459efe-bece-4aba-86e8-76fc06de16fb: !Template - answer_choices: null - id: c6459efe-bece-4aba-86e8-76fc06de16fb + a9f837e3-4a75-48b5-8fcf-4f18cb3e329d: !Template + answer_choices: A ||| B + id: a9f837e3-4a75-48b5-8fcf-4f18cb3e329d jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -212,11 +190,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -224,15 +202,15 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: grammatical_between_A_B reference: '' - cefda071-1c35-4fce-9577-c63c7f3f0fde: !Template + ad3d2795-6613-4392-bc1d-13f774928042: !Template answer_choices: Yes ||| No - id: cefda071-1c35-4fce-9577-c63c7f3f0fde + id: ad3d2795-6613-4392-bc1d-13f774928042 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata @@ -240,54 +218,76 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' + name: single_sentence_bad_no_yes reference: '' - ddd7ada0-6cb7-448a-85c4-fe3505509bcc: !Template - answer_choices: Yes ||| No - id: ddd7ada0-6cb7-448a-85c4-fe3505509bcc - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + ae60cb40-3c68-491d-98fa-b92af9291d9d: !Template + answer_choices: A ||| B + id: ae60cb40-3c68-491d-98fa-b92af9291d9d + jinja: '{% set shuffled_order = [0, 1] | random %} - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' + original_task: true + name: grammatical_which_one_A_B reference: '' - e6ae10f4-f1dd-4733-a806-9e96664ed8ed: !Template - answer_choices: null - id: e6ae10f4-f1dd-4733-a806-9e96664ed8ed + c329889d-2046-4ab7-bae8-abd4bbc639fd: !Template + answer_choices: A ||| B + id: c329889d-2046-4ab7-bae8-abd4bbc639fd jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -295,5 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: better_B_A reference: '' diff --git a/promptsource/templates/blimp/irregular_past_participle_adjectives/templates.yaml b/promptsource/templates/blimp/irregular_past_participle_adjectives/templates.yaml index 2dd39a9bc..439abf7d3 100644 --- a/promptsource/templates/blimp/irregular_past_participle_adjectives/templates.yaml +++ b/promptsource/templates/blimp/irregular_past_participle_adjectives/templates.yaml @@ -1,9 +1,9 @@ dataset: blimp subset: irregular_past_participle_adjectives templates: - 1a01ffb7-ba38-4dcd-95fe-154fd1f2e1e6: !Template + 23bc54ae-098c-4bf1-ad60-a6cc489395f4: !Template answer_choices: A ||| B - id: 1a01ffb7-ba38-4dcd-95fe-154fd1f2e1e6 + id: 23bc54ae-098c-4bf1-ad60-a6cc489395f4 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -37,26 +37,26 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: A-B)' + name: better_A_B reference: '' - 4091948a-5bcd-4dee-96c7-8d01097cc60d: !Template - answer_choices: null - id: 4091948a-5bcd-4dee-96c7-8d01097cc60d + 26a47924-2a4c-4c1e-92f6-22e78274bcc9: !Template + answer_choices: A ||| B + id: 26a47924-2a4c-4c1e-92f6-22e78274bcc9 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -64,11 +64,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -76,15 +76,15 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: grammatical_which_one_A_B reference: '' - 47d0cdaf-8702-42f6-b998-919b845a310e: !Template + 57f5a493-b0e2-4846-869a-5a69e5e7455e: !Template answer_choices: Yes ||| No - id: 47d0cdaf-8702-42f6-b998-919b845a310e + id: 57f5a493-b0e2-4846-869a-5a69e5e7455e jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -92,38 +92,53 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' + name: single_sentence_good_yes_no reference: '' - 7375cdca-4772-4e31-9e7e-e1b7bf5c09b2: !Template - answer_choices: null - id: 7375cdca-4772-4e31-9e7e-e1b7bf5c09b2 + 5935513b-f7cf-4acf-8fcb-4d12b3e0c8d0: !Template + answer_choices: Yes ||| No + id: 5935513b-f7cf-4acf-8fcb-4d12b3e0c8d0 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes + reference: '' + 65c4f840-4084-4526-af02-2f26edb24172: !Template + answer_choices: A ||| B + id: 65c4f840-4084-4526-af02-2f26edb24172 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -131,42 +146,26 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 - reference: '' - 7a89daa4-c45e-4d7b-ad7c-55de85a2ffcd: !Template - answer_choices: Yes ||| No - id: 7a89daa4-c45e-4d7b-ad7c-55de85a2ffcd - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' + name: better_B_A reference: '' - a292d01a-f262-449f-b1d4-979fcfcb4ad2: !Template - answer_choices: null - id: a292d01a-f262-449f-b1d4-979fcfcb4ad2 + 6fc75db1-2999-4a0d-964b-3fc1d65f7e55: !Template + answer_choices: 1 ||| 2 + id: 6fc75db1-2999-4a0d-964b-3fc1d65f7e55 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -174,11 +173,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -186,15 +185,15 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: grammatical_which_one_1_2 reference: '' - a732517d-7970-4640-9e2c-f66866adbc02: !Template + 834fd501-fb19-449d-8f44-65ae2423904b: !Template answer_choices: Yes ||| No - id: a732517d-7970-4640-9e2c-f66866adbc02 + id: 834fd501-fb19-449d-8f44-65ae2423904b jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -202,15 +201,15 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' + name: single_sentence_good_no_yes reference: '' - b8a31816-9fa3-483d-ae32-5e694e5c7e1c: !Template + 8fcbb887-6c8d-4d85-a7ef-839db220d0c0: !Template answer_choices: Yes ||| No - id: b8a31816-9fa3-483d-ae32-5e694e5c7e1c + id: 8fcbb887-6c8d-4d85-a7ef-839db220d0c0 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata @@ -218,31 +217,32 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' + name: single_sentence_bad_yes_no reference: '' - bd4ef1c2-c5eb-420b-897a-ee4d2102657d: !Template + b2ed7261-81f0-48ec-8f1e-b8fa01cfff6e: !Template answer_choices: A ||| B - id: bd4ef1c2-c5eb-420b-897a-ee4d2102657d + id: b2ed7261-81f0-48ec-8f1e-b8fa01cfff6e jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -256,26 +256,26 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: B-A)' + name: grammatical_between_A_B reference: '' - bedfb44d-0bdd-469e-a57b-ab2c531edd45: !Template - answer_choices: null - id: bedfb44d-0bdd-469e-a57b-ab2c531edd45 + edf23807-c574-4925-92cd-b2799a08bc4e: !Template + answer_choices: 1 ||| 2 + id: edf23807-c574-4925-92cd-b2799a08bc4e jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -283,11 +283,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -295,5 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: grammatical_between_1_2 reference: '' diff --git a/promptsource/templates/blimp/irregular_past_participle_verbs/templates.yaml b/promptsource/templates/blimp/irregular_past_participle_verbs/templates.yaml index 35aa8daae..88f2db717 100644 --- a/promptsource/templates/blimp/irregular_past_participle_verbs/templates.yaml +++ b/promptsource/templates/blimp/irregular_past_participle_verbs/templates.yaml @@ -1,25 +1,47 @@ dataset: blimp subset: irregular_past_participle_verbs templates: - 05d49540-bde0-46b8-801d-b8cde4c28e7f: !Template - answer_choices: Yes ||| No - id: 05d49540-bde0-46b8-801d-b8cde4c28e7f - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + 11db3871-f573-4089-b0d2-0b8fff3b0b8c: !Template + answer_choices: A ||| B + id: 11db3871-f573-4089-b0d2-0b8fff3b0b8c + jinja: '{% set shuffled_order = [0, 1] | random %} - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' + original_task: true + name: better_A_B reference: '' - 07f426ef-3848-4175-91b5-070199cff944: !Template - answer_choices: null - id: 07f426ef-3848-4175-91b5-070199cff944 + 23f44d7d-cdd9-4194-8acc-b65205aed9b9: !Template + answer_choices: A ||| B + id: 23f44d7d-cdd9-4194-8acc-b65205aed9b9 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer A or B. @@ -42,11 +64,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -54,26 +76,26 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: grammatical_which_one_A_B reference: '' - 39797145-c279-4a20-9422-842a1af6e139: !Template - answer_choices: null - id: 39797145-c279-4a20-9422-842a1af6e139 + 2ea2f62f-6940-406e-9f0d-126b714c4d5b: !Template + answer_choices: A ||| B + id: 2ea2f62f-6940-406e-9f0d-126b714c4d5b jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -81,11 +103,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -93,11 +115,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: grammatical_between_A_B reference: '' - 6bdb527d-4ac2-4aac-bf46-52d823318b10: !Template + 3c3f23a1-ce5c-4219-befe-f613a054ed13: !Template answer_choices: Yes ||| No - id: 6bdb527d-4ac2-4aac-bf46-52d823318b10 + id: 3c3f23a1-ce5c-4219-befe-f613a054ed13 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -109,11 +131,11 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' + name: single_sentence_good_yes_no reference: '' - 98bc55d7-093f-44ba-bdf4-7481579675db: !Template + 41bb15eb-77af-402b-9f30-e23f5eb39060: !Template answer_choices: A ||| B - id: 98bc55d7-093f-44ba-bdf4-7481579675db + id: 41bb15eb-77af-402b-9f30-e23f5eb39060 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -132,7 +154,7 @@ templates: {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order + Which one is a better sentence of English, B or A? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -147,26 +169,42 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: A-B)' + name: better_B_A + reference: '' + 5b0ffc77-aac1-4aba-a4ca-d6217c02f690: !Template + answer_choices: Yes ||| No + id: 5b0ffc77-aac1-4aba-a4ca-d6217c02f690 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no reference: '' - a50db2db-154e-4e0b-87f3-5c1a453e39ff: !Template - answer_choices: null - id: a50db2db-154e-4e0b-87f3-5c1a453e39ff + 7917ccfe-7790-4969-bc66-0e2f30690088: !Template + answer_choices: 1 ||| 2 + id: 7917ccfe-7790-4969-bc66-0e2f30690088 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -174,11 +212,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -186,65 +224,43 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: grammatical_which_one_1_2 reference: '' - b1bb44fc-fb4d-49a4-b149-21bd6b2e0432: !Template + b0a3745d-c47d-44c8-b730-c1cb6ee527ac: !Template answer_choices: Yes ||| No - id: b1bb44fc-fb4d-49a4-b149-21bd6b2e0432 + id: b0a3745d-c47d-44c8-b730-c1cb6ee527ac jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' + name: single_sentence_bad_no_yes reference: '' - bfb444a9-cc45-47dc-b041-c75713b996bc: !Template - answer_choices: A ||| B - id: bfb444a9-cc45-47dc-b041-c75713b996bc - jinja: '{% set shuffled_order = [0, 1] | random %} - - Read the following two sentences. - - {% if shuffled_order == 0 %} - - Sentence A: {{ sentence_good }} - - Sentence B: {{ sentence_bad }} - - {% else %} - - Sentence A: {{ sentence_bad }} - - Sentence B: {{ sentence_good }} - - {% endif %} - - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} + cbb18fd4-d194-4f2c-b34b-99c8a2b7b77c: !Template + answer_choices: Yes ||| No + id: cbb18fd4-d194-4f2c-b34b-99c8a2b7b77c + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - {% endif %}' + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: 'A/B choice randomized (choice order: B-A)' + original_task: false + name: single_sentence_good_no_yes reference: '' - d54c908d-ad8b-4c97-8f12-d903a2be93e8: !Template - answer_choices: null - id: d54c908d-ad8b-4c97-8f12-d903a2be93e8 + fbe74e44-6434-4389-94de-5360a1fde011: !Template + answer_choices: 1 ||| 2 + id: fbe74e44-6434-4389-94de-5360a1fde011 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -267,11 +283,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -279,21 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 - reference: '' - db009dc0-b4c3-4067-a421-9a68cdf96bad: !Template - answer_choices: Yes ||| No - id: db009dc0-b4c3-4067-a421-9a68cdf96bad - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' + name: grammatical_between_1_2 reference: '' diff --git a/promptsource/templates/blimp/irregular_plural_subject_verb_agreement_1/templates.yaml b/promptsource/templates/blimp/irregular_plural_subject_verb_agreement_1/templates.yaml index db5c18280..702c01c9d 100644 --- a/promptsource/templates/blimp/irregular_plural_subject_verb_agreement_1/templates.yaml +++ b/promptsource/templates/blimp/irregular_plural_subject_verb_agreement_1/templates.yaml @@ -1,52 +1,35 @@ dataset: blimp subset: irregular_plural_subject_verb_agreement_1 templates: - 0f0fd016-1f19-4715-8e05-9ff549088767: !Template - answer_choices: Yes ||| No - id: 0f0fd016-1f19-4715-8e05-9ff549088767 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' - reference: '' - 222939ba-e155-4ff3-9ae7-ff1cd5c2713e: !Template - answer_choices: null - id: 222939ba-e155-4ff3-9ae7-ff1cd5c2713e + 24a53c55-2980-4599-b329-5ab988dd7b2a: !Template + answer_choices: A ||| B + id: 24a53c55-2980-4599-b329-5ab988dd7b2a jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -54,31 +37,32 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: better_A_B reference: '' - 3506c95f-a239-4eaf-ad92-62e9ff23d3db: !Template - answer_choices: A ||| B - id: 3506c95f-a239-4eaf-ad92-62e9ff23d3db + 3a947519-9297-4256-8aee-075be3060c3e: !Template + answer_choices: 1 ||| 2 + id: 3a947519-9297-4256-8aee-075be3060c3e jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -92,31 +76,32 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: A-B)' + name: grammatical_which_one_1_2 reference: '' - 355c9af4-4aff-4039-b35d-b9fc04893375: !Template + 438261f2-b8d6-4554-8b37-23427a9b5a4e: !Template answer_choices: A ||| B - id: 355c9af4-4aff-4039-b35d-b9fc04893375 + id: 438261f2-b8d6-4554-8b37-23427a9b5a4e jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -130,15 +115,15 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: B-A)' + name: grammatical_between_A_B reference: '' - 523dcb7c-8a66-46de-8d80-0d18f53583b5: !Template + 91735f88-fd6b-45ed-8730-1dc85fcc1fb3: !Template answer_choices: Yes ||| No - id: 523dcb7c-8a66-46de-8d80-0d18f53583b5 + id: 91735f88-fd6b-45ed-8730-1dc85fcc1fb3 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -146,26 +131,26 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' + name: single_sentence_good_yes_no reference: '' - 56d641b8-b550-454a-8b50-2b8637ed78b0: !Template - answer_choices: null - id: 56d641b8-b550-454a-8b50-2b8637ed78b0 + 960258d2-0bbe-4f6c-a7ca-85fa7a5efbfd: !Template + answer_choices: A ||| B + id: 960258d2-0bbe-4f6c-a7ca-85fa7a5efbfd jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -173,11 +158,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -185,15 +170,15 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: grammatical_which_one_A_B reference: '' - 5fcd802b-29d4-417c-9871-6e7337f56f2c: !Template + c60e7e92-1538-4638-bf8a-5c70739bea92: !Template answer_choices: Yes ||| No - id: 5fcd802b-29d4-417c-9871-6e7337f56f2c + id: c60e7e92-1538-4638-bf8a-5c70739bea92 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -201,26 +186,42 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' + name: single_sentence_good_no_yes reference: '' - 613c8f2e-8c3d-41f2-b1bc-d2fd705ef80d: !Template - answer_choices: null - id: 613c8f2e-8c3d-41f2-b1bc-d2fd705ef80d + de803bcc-9767-47cc-a176-a175ddc9fce6: !Template + answer_choices: Yes ||| No + id: de803bcc-9767-47cc-a176-a175ddc9fce6 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes + reference: '' + e51d328a-cd40-4be0-af2a-5c50b32ee303: !Template + answer_choices: 1 ||| 2 + id: e51d328a-cd40-4be0-af2a-5c50b32ee303 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -228,11 +229,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -240,38 +241,37 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: grammatical_between_1_2 reference: '' - 619c0b3d-d914-42f3-af5a-3da0ed47fd61: !Template - answer_choices: null - id: 619c0b3d-d914-42f3-af5a-3da0ed47fd61 + ecc18041-dd7e-44a2-8a1c-a376faeab14c: !Template + answer_choices: A ||| B + id: ecc18041-dd7e-44a2-8a1c-a376faeab14c jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -279,15 +279,15 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: better_B_A reference: '' - 6cc4b448-6fb7-4385-baba-6ff904dfcc9e: !Template + eef40269-9104-453c-a521-27dae1b187d9: !Template answer_choices: Yes ||| No - id: 6cc4b448-6fb7-4385-baba-6ff904dfcc9e + id: eef40269-9104-453c-a521-27dae1b187d9 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata @@ -295,5 +295,5 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' + name: single_sentence_bad_yes_no reference: '' diff --git a/promptsource/templates/blimp/irregular_plural_subject_verb_agreement_2/templates.yaml b/promptsource/templates/blimp/irregular_plural_subject_verb_agreement_2/templates.yaml index 1f09dec8d..790b8615a 100644 --- a/promptsource/templates/blimp/irregular_plural_subject_verb_agreement_2/templates.yaml +++ b/promptsource/templates/blimp/irregular_plural_subject_verb_agreement_2/templates.yaml @@ -1,29 +1,30 @@ dataset: blimp subset: irregular_plural_subject_verb_agreement_2 templates: - 14f0a497-ffae-486e-87b9-e114a4395ada: !Template - answer_choices: A ||| B - id: 14f0a497-ffae-486e-87b9-e114a4395ada + 1f60b3c6-01fa-45e9-bb04-d8d57a9e770e: !Template + answer_choices: 1 ||| 2 + id: 1f60b3c6-01fa-45e9-bb04-d8d57a9e770e jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -37,26 +38,26 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: B-A)' + name: grammatical_between_1_2 reference: '' - 20f3b2ee-426e-4999-aa43-ae903a028348: !Template - answer_choices: null - id: 20f3b2ee-426e-4999-aa43-ae903a028348 + 43af7924-0682-4f49-8fc6-acc3620a62f0: !Template + answer_choices: 1 ||| 2 + id: 43af7924-0682-4f49-8fc6-acc3620a62f0 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -64,11 +65,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -76,11 +77,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: grammatical_which_one_1_2 reference: '' - 3e5bec52-b152-495e-8bf0-f6462997a455: !Template + 7f94f3a2-6c03-420b-a363-75cd2c95c05f: !Template answer_choices: Yes ||| No - id: 3e5bec52-b152-495e-8bf0-f6462997a455 + id: 7f94f3a2-6c03-420b-a363-75cd2c95c05f jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -92,26 +93,26 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' + name: single_sentence_good_yes_no reference: '' - 4d03ecc5-9fee-487e-bd8d-b079731cc239: !Template - answer_choices: null - id: 4d03ecc5-9fee-487e-bd8d-b079731cc239 + 8bec548b-cbd6-462d-92bb-6034bf57bc65: !Template + answer_choices: A ||| B + id: 8bec548b-cbd6-462d-92bb-6034bf57bc65 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -119,11 +120,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -131,14 +132,14 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: grammatical_between_A_B reference: '' - 59cc4a6a-178d-4cce-93dd-d586a810056e: !Template - answer_choices: null - id: 59cc4a6a-178d-4cce-93dd-d586a810056e + a274f8b4-f55f-436d-bb37-f0ed93a5eb17: !Template + answer_choices: A ||| B + id: a274f8b4-f55f-436d-bb37-f0ed93a5eb17 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -158,11 +159,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -170,11 +171,43 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: grammatical_which_one_A_B + reference: '' + a96f8259-6df9-4d65-b8a1-53894a99c0c6: !Template + answer_choices: Yes ||| No + id: a96f8259-6df9-4d65-b8a1-53894a99c0c6 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no reference: '' - 7e00c53f-3cc4-4747-b501-795a3f016966: !Template + bed5216d-07b3-4f08-8927-c1008929a95a: !Template + answer_choices: Yes ||| No + id: bed5216d-07b3-4f08-8927-c1008929a95a + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes + reference: '' + c75dd4fd-efed-4b14-8987-2fd4f429c1b2: !Template answer_choices: A ||| B - id: 7e00c53f-3cc4-4747-b501-795a3f016966 + id: c75dd4fd-efed-4b14-8987-2fd4f429c1b2 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -208,27 +241,11 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: A-B)' + name: better_A_B reference: '' - 7eded14d-5d1b-40a4-ba11-a9d45feaaa79: !Template + d88168a9-462e-455f-b5c5-cbe5dd22b449: !Template answer_choices: Yes ||| No - id: 7eded14d-5d1b-40a4-ba11-a9d45feaaa79 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' - reference: '' - 7f7484bc-3faf-4462-b453-321c0943b241: !Template - answer_choices: Yes ||| No - id: 7f7484bc-3faf-4462-b453-321c0943b241 + id: d88168a9-462e-455f-b5c5-cbe5dd22b449 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -240,54 +257,37 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' - reference: '' - aee62b04-04c9-4267-ab36-03e1b79415b2: !Template - answer_choices: Yes ||| No - id: aee62b04-04c9-4267-ab36-03e1b79415b2 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' + name: single_sentence_bad_no_yes reference: '' - e7ea957b-df45-4e9b-804e-975e9ea50c8d: !Template - answer_choices: null - id: e7ea957b-df45-4e9b-804e-975e9ea50c8d + f47179e9-df3e-4890-b722-2d8438724c49: !Template + answer_choices: A ||| B + id: f47179e9-df3e-4890-b722-2d8438724c49 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -295,5 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: better_B_A reference: '' diff --git a/promptsource/templates/blimp/left_branch_island_echo_question/templates.yaml b/promptsource/templates/blimp/left_branch_island_echo_question/templates.yaml index 7d55c8625..566b79789 100644 --- a/promptsource/templates/blimp/left_branch_island_echo_question/templates.yaml +++ b/promptsource/templates/blimp/left_branch_island_echo_question/templates.yaml @@ -1,45 +1,30 @@ dataset: blimp subset: left_branch_island_echo_question templates: - 0cf0efad-670f-4dc2-95a5-6a151ce47bec: !Template - answer_choices: Yes ||| No - id: 0cf0efad-670f-4dc2-95a5-6a151ce47bec - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' - reference: '' - 3be69c93-e8a3-4204-9d62-325c6e50398a: !Template + 7523e51d-248c-4965-a94a-e7155bf752df: !Template answer_choices: A ||| B - id: 3be69c93-e8a3-4204-9d62-325c6e50398a + id: 7523e51d-248c-4965-a94a-e7155bf752df jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -53,31 +38,32 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: A-B)' + name: grammatical_between_A_B reference: '' - 451b85b0-b3ad-4279-b4e7-62f7a0ee4d5a: !Template - answer_choices: A ||| B - id: 451b85b0-b3ad-4279-b4e7-62f7a0ee4d5a + 79613311-d2b3-41b7-aaf3-f3e5a99fd63c: !Template + answer_choices: 1 ||| 2 + id: 79613311-d2b3-41b7-aaf3-f3e5a99fd63c jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -91,11 +77,27 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: B-A)' + name: grammatical_which_one_1_2 reference: '' - 5f40393a-be79-4dad-98bb-30f0144c5b2e: !Template - answer_choices: null - id: 5f40393a-be79-4dad-98bb-30f0144c5b2e + 9b60e503-bf0c-4d5a-949b-ae81af206854: !Template + answer_choices: Yes ||| No + id: 9b60e503-bf0c-4d5a-949b-ae81af206854 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + a7daf9c4-c4a3-4650-af89-e63370c96ac8: !Template + answer_choices: A ||| B + id: a7daf9c4-c4a3-4650-af89-e63370c96ac8 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer A or B. @@ -118,11 +120,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -130,54 +132,53 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: grammatical_which_one_A_B reference: '' - 70bcf6ab-3460-44d5-acba-0f5648868f9e: !Template + abcf99e4-c6b7-4408-be5b-7c35e84b6b41: !Template answer_choices: Yes ||| No - id: 70bcf6ab-3460-44d5-acba-0f5648868f9e + id: abcf99e4-c6b7-4408-be5b-7c35e84b6b41 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' + name: single_sentence_bad_no_yes reference: '' - 80f41567-8aef-4c05-867b-5555912d6c7d: !Template - answer_choices: null - id: 80f41567-8aef-4c05-867b-5555912d6c7d + c4e5cd99-6d94-436d-b171-68652a131c3e: !Template + answer_choices: A ||| B + id: c4e5cd99-6d94-436d-b171-68652a131c3e jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -185,38 +186,53 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: better_B_A reference: '' - c15a84e8-96d6-48c7-aa02-83f5e935c9fc: !Template - answer_choices: null - id: c15a84e8-96d6-48c7-aa02-83f5e935c9fc + db819087-a96c-487d-aec7-2e623a22c3f4: !Template + answer_choices: Yes ||| No + id: db819087-a96c-487d-aec7-2e623a22c3f4 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + e27def85-1590-4c28-ad69-9a7d89c83b54: !Template + answer_choices: A ||| B + id: e27def85-1590-4c28-ad69-9a7d89c83b54 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -224,27 +240,27 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: better_A_B reference: '' - c9ae4c70-f150-401a-bfa7-67794a694d0b: !Template + e55d47cb-f5a0-494f-8f2d-d558e4dd3cca: !Template answer_choices: Yes ||| No - id: c9ae4c70-f150-401a-bfa7-67794a694d0b + id: e55d47cb-f5a0-494f-8f2d-d558e4dd3cca jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' + name: single_sentence_good_no_yes reference: '' - dafde0f0-b07a-435c-be05-1d28a3a26edc: !Template - answer_choices: null - id: dafde0f0-b07a-435c-be05-1d28a3a26edc + fab92cfb-355c-4c52-b0a4-b2d68c47cf84: !Template + answer_choices: 1 ||| 2 + id: fab92cfb-355c-4c52-b0a4-b2d68c47cf84 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -267,11 +283,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -279,21 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 - reference: '' - e039d66f-3dfa-4451-9102-055ad545f55b: !Template - answer_choices: Yes ||| No - id: e039d66f-3dfa-4451-9102-055ad545f55b - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' + name: grammatical_between_1_2 reference: '' diff --git a/promptsource/templates/blimp/left_branch_island_simple_question/templates.yaml b/promptsource/templates/blimp/left_branch_island_simple_question/templates.yaml index 81f03a0bf..082474b5a 100644 --- a/promptsource/templates/blimp/left_branch_island_simple_question/templates.yaml +++ b/promptsource/templates/blimp/left_branch_island_simple_question/templates.yaml @@ -1,9 +1,9 @@ dataset: blimp subset: left_branch_island_simple_question templates: - 17335753-95c8-4d9f-ad1a-a13234d930aa: !Template + 015c5654-d491-4f48-be9e-4e5f5da0451c: !Template answer_choices: Yes ||| No - id: 17335753-95c8-4d9f-ad1a-a13234d930aa + id: 015c5654-d491-4f48-be9e-4e5f5da0451c jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -15,15 +15,15 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' + name: single_sentence_good_no_yes reference: '' - 1b58fda4-95bc-44f6-a4cb-92c2f91d313b: !Template + 212f7bdb-78ef-4fa8-b418-527638cebdeb: !Template answer_choices: Yes ||| No - id: 1b58fda4-95bc-44f6-a4cb-92c2f91d313b + id: 212f7bdb-78ef-4fa8-b418-527638cebdeb jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata @@ -31,42 +31,64 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' + name: single_sentence_bad_no_yes reference: '' - 5967ff7d-1b12-463c-b8e5-a8e20a4bfeba: !Template - answer_choices: Yes ||| No - id: 5967ff7d-1b12-463c-b8e5-a8e20a4bfeba - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + 2e2af07a-50b7-4579-9953-6d485d14acf4: !Template + answer_choices: A ||| B + id: 2e2af07a-50b7-4579-9953-6d485d14acf4 + jinja: '{% set shuffled_order = [0, 1] | random %} - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' + original_task: true + name: better_A_B reference: '' - 614f74b1-9130-4c89-b65b-8e8efa3069bd: !Template - answer_choices: null - id: 614f74b1-9130-4c89-b65b-8e8efa3069bd + 322706b1-4ccd-46c3-9cd1-a17cc4bc1b4c: !Template + answer_choices: 1 ||| 2 + id: 322706b1-4ccd-46c3-9cd1-a17cc4bc1b4c jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -74,11 +96,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -86,14 +108,14 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: grammatical_between_1_2 reference: '' - 7fec3889-4649-4c78-8633-430c3683c847: !Template - answer_choices: null - id: 7fec3889-4649-4c78-8633-430c3683c847 + 686f3007-e2b2-4d35-8d0d-3c4b8a61f300: !Template + answer_choices: A ||| B + id: 686f3007-e2b2-4d35-8d0d-3c4b8a61f300 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -113,11 +135,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -125,31 +147,32 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: grammatical_between_A_B reference: '' - 830edf75-0c8d-4ed3-ada7-af7b6715fb24: !Template - answer_choices: A ||| B - id: 830edf75-0c8d-4ed3-ada7-af7b6715fb24 + 83dfb5ab-4462-4ce3-af1c-76e549ee03eb: !Template + answer_choices: 1 ||| 2 + id: 83dfb5ab-4462-4ce3-af1c-76e549ee03eb jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -163,47 +186,48 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: B-A)' + name: grammatical_which_one_1_2 reference: '' - 99875376-fa46-454a-a2bb-0ea56e0beb21: !Template + 99d6cb21-a620-42e4-9821-36a0e3c0eb1c: !Template answer_choices: Yes ||| No - id: 99875376-fa46-454a-a2bb-0ea56e0beb21 + id: 99d6cb21-a620-42e4-9821-36a0e3c0eb1c jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' + name: single_sentence_good_yes_no reference: '' - a91945c6-5318-4154-866e-85904700b684: !Template + d6493bf6-1d09-4b6c-b908-eea6ffc7d909: !Template answer_choices: A ||| B - id: a91945c6-5318-4154-866e-85904700b684 + id: d6493bf6-1d09-4b6c-b908-eea6ffc7d909 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -217,38 +241,37 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: A-B)' + name: grammatical_which_one_A_B reference: '' - e336851f-5d24-496c-90e1-441f602384eb: !Template - answer_choices: null - id: e336851f-5d24-496c-90e1-441f602384eb + ef134e9a-c50f-45a5-a49b-d56a408d1332: !Template + answer_choices: A ||| B + id: ef134e9a-c50f-45a5-a49b-d56a408d1332 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -256,44 +279,21 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: better_B_A reference: '' - fdaa2577-c1ea-4b60-bfa2-3c58cca94bfc: !Template - answer_choices: null - id: fdaa2577-c1ea-4b60-bfa2-3c58cca94bfc - jinja: '{% set shuffled_order = [0, 1] | random %} - - Which one of the following sentences is grammatical? Please answer 1 or 2. - - {% if shuffled_order == 0 %} - - 1: {{ sentence_good }} - - 2: {{ sentence_bad }} - - {% else %} - - 1: {{ sentence_bad }} - - 2: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - 1 - - {% else %} - - 2 + f39202c2-c4af-4415-b9ff-5dbf2003a967: !Template + answer_choices: Yes ||| No + id: f39202c2-c4af-4415-b9ff-5dbf2003a967 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - {% endif %}' + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: random_choice_1_2 + original_task: false + name: single_sentence_bad_yes_no reference: '' diff --git a/promptsource/templates/blimp/matrix_question_npi_licensor_present/templates.yaml b/promptsource/templates/blimp/matrix_question_npi_licensor_present/templates.yaml index e8c7851df..12d5ec4ef 100644 --- a/promptsource/templates/blimp/matrix_question_npi_licensor_present/templates.yaml +++ b/promptsource/templates/blimp/matrix_question_npi_licensor_present/templates.yaml @@ -1,9 +1,9 @@ dataset: blimp subset: matrix_question_npi_licensor_present templates: - 1c40ac09-5369-4a9e-a44c-14319215613b: !Template + 1bcb85f9-7794-45b2-a562-676bf5fa3afe: !Template answer_choices: Yes ||| No - id: 1c40ac09-5369-4a9e-a44c-14319215613b + id: 1bcb85f9-7794-45b2-a562-676bf5fa3afe jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -15,11 +15,11 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' + name: single_sentence_good_yes_no reference: '' - 28e46e34-0ee1-481b-93f6-010fd2a1c144: !Template + 252187ac-19ca-4c36-b201-0d15d0403972: !Template answer_choices: A ||| B - id: 28e46e34-0ee1-481b-93f6-010fd2a1c144 + id: 252187ac-19ca-4c36-b201-0d15d0403972 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -38,7 +38,7 @@ templates: {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order + Which one is a better sentence of English, A or B? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -53,31 +53,32 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: B-A)' + name: better_A_B reference: '' - 387892c8-c6fc-4110-a460-096ced7092a5: !Template + 597931f7-e460-46de-847f-dac2d26a643e: !Template answer_choices: A ||| B - id: 387892c8-c6fc-4110-a460-096ced7092a5 + id: 597931f7-e460-46de-847f-dac2d26a643e jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -91,74 +92,26 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: A-B)' - reference: '' - 62df645b-2a76-431b-92f3-8c2793701300: !Template - answer_choices: Yes ||| No - id: 62df645b-2a76-431b-92f3-8c2793701300 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' - reference: '' - 63ad3ddd-d5eb-4e55-9830-c0dedae59f86: !Template - answer_choices: Yes ||| No - id: 63ad3ddd-d5eb-4e55-9830-c0dedae59f86 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' + name: grammatical_between_A_B reference: '' - beea3fb1-eb4e-48d0-aab6-5b26f8e606e2: !Template - answer_choices: Yes ||| No - id: beea3fb1-eb4e-48d0-aab6-5b26f8e606e2 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' - reference: '' - c05dd57c-7874-4f05-a5f2-0794ed167cdf: !Template - answer_choices: null - id: c05dd57c-7874-4f05-a5f2-0794ed167cdf + 6079f676-84cd-4683-98b9-1ac9e2d61980: !Template + answer_choices: A ||| B + id: 6079f676-84cd-4683-98b9-1ac9e2d61980 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -166,11 +119,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -178,38 +131,37 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: grammatical_which_one_A_B reference: '' - ccde093e-c789-459d-bed9-4967b9b359a4: !Template - answer_choices: null - id: ccde093e-c789-459d-bed9-4967b9b359a4 + 7a7b4b49-9194-41d7-9c86-04ebae2ec3f7: !Template + answer_choices: A ||| B + id: 7a7b4b49-9194-41d7-9c86-04ebae2ec3f7 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -217,26 +169,42 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: better_B_A reference: '' - d60097f5-215c-43cc-a78f-9ccca5db2adc: !Template - answer_choices: null - id: d60097f5-215c-43cc-a78f-9ccca5db2adc + 951c8b6d-e13b-41aa-bd13-04d2965491db: !Template + answer_choices: Yes ||| No + id: 951c8b6d-e13b-41aa-bd13-04d2965491db + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes + reference: '' + a4079add-50bb-4530-a7d6-3807d6b64052: !Template + answer_choices: 1 ||| 2 + id: a4079add-50bb-4530-a7d6-3807d6b64052 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -244,11 +212,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -256,11 +224,27 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: grammatical_between_1_2 reference: '' - da633e5d-adb4-4027-9204-6bf529dbd199: !Template - answer_choices: null - id: da633e5d-adb4-4027-9204-6bf529dbd199 + c1f1320d-8dd0-499b-97f4-004201f390f7: !Template + answer_choices: Yes ||| No + id: c1f1320d-8dd0-499b-97f4-004201f390f7 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes + reference: '' + c3bf87c9-06a7-4e51-abb0-6f51a9b62402: !Template + answer_choices: 1 ||| 2 + id: c3bf87c9-06a7-4e51-abb0-6f51a9b62402 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer 1 or 2. @@ -283,11 +267,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -295,5 +279,21 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: grammatical_which_one_1_2 + reference: '' + f9fd9e89-7c1a-4509-bda6-fd5642715cda: !Template + answer_choices: Yes ||| No + id: f9fd9e89-7c1a-4509-bda6-fd5642715cda + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no reference: '' diff --git a/promptsource/templates/blimp/npi_present_1/templates.yaml b/promptsource/templates/blimp/npi_present_1/templates.yaml index 5dd7e9e0d..c1697ab99 100644 --- a/promptsource/templates/blimp/npi_present_1/templates.yaml +++ b/promptsource/templates/blimp/npi_present_1/templates.yaml @@ -1,61 +1,30 @@ dataset: blimp subset: npi_present_1 templates: - 14429731-f9a5-4a5e-8a0e-eddde4cf8e24: !Template - answer_choices: Yes ||| No - id: 14429731-f9a5-4a5e-8a0e-eddde4cf8e24 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' - reference: '' - 1a7d2011-76d3-486d-8ab7-021721cc72a1: !Template - answer_choices: Yes ||| No - id: 1a7d2011-76d3-486d-8ab7-021721cc72a1 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' - reference: '' - 92d739a8-fda7-4fc7-83be-560c2b913702: !Template - answer_choices: A ||| B - id: 92d739a8-fda7-4fc7-83be-560c2b913702 + 32056d72-aea1-4e8d-ad7c-0f81b780cf37: !Template + answer_choices: 1 ||| 2 + id: 32056d72-aea1-4e8d-ad7c-0f81b780cf37 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -69,38 +38,37 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: B-A)' + name: grammatical_which_one_1_2 reference: '' - 95500764-37bd-466e-9bd3-c4bb179f79c8: !Template - answer_choices: null - id: 95500764-37bd-466e-9bd3-c4bb179f79c8 + 32f4aadc-ba57-4d97-bd43-985c7447ede6: !Template + answer_choices: A ||| B + id: 32f4aadc-ba57-4d97-bd43-985c7447ede6 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -108,11 +76,27 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: better_B_A + reference: '' + 3736904c-b062-402d-acc1-8c946788c9dd: !Template + answer_choices: Yes ||| No + id: 3736904c-b062-402d-acc1-8c946788c9dd + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no reference: '' - bb05b582-e555-4a77-9a1a-c98e58fd5784: !Template + 465acd78-1df0-4490-8c22-86f39387cd35: !Template answer_choices: A ||| B - id: bb05b582-e555-4a77-9a1a-c98e58fd5784 + id: 465acd78-1df0-4490-8c22-86f39387cd35 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -146,14 +130,14 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: A-B)' + name: better_A_B reference: '' - c1c38791-307e-4aa7-a317-6152d3eafe53: !Template - answer_choices: null - id: c1c38791-307e-4aa7-a317-6152d3eafe53 + 532862c3-b1ad-44a1-b819-2fbf4a3f3932: !Template + answer_choices: A ||| B + id: 532862c3-b1ad-44a1-b819-2fbf4a3f3932 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -173,11 +157,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -185,26 +169,42 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: grammatical_which_one_A_B + reference: '' + 76a64f9b-a4db-4099-afc0-4cdcc9c741b8: !Template + answer_choices: Yes ||| No + id: 76a64f9b-a4db-4099-afc0-4cdcc9c741b8 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no reference: '' - ce21e2be-b2e3-4920-941e-5e8e5c3eff2d: !Template - answer_choices: null - id: ce21e2be-b2e3-4920-941e-5e8e5c3eff2d + 9eb6c3ab-5682-40d0-8576-462c27b33d95: !Template + answer_choices: A ||| B + id: 9eb6c3ab-5682-40d0-8576-462c27b33d95 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -212,11 +212,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -224,15 +224,15 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: grammatical_between_A_B reference: '' - dcb03908-30ba-4089-a3ed-9e0754bfee35: !Template + a5d17a66-e645-4eaf-a2aa-ef8f7abc9dda: !Template answer_choices: Yes ||| No - id: dcb03908-30ba-4089-a3ed-9e0754bfee35 + id: a5d17a66-e645-4eaf-a2aa-ef8f7abc9dda jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata @@ -240,26 +240,42 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' + name: single_sentence_bad_no_yes + reference: '' + ca363a08-c111-42d6-b643-273f2ec56249: !Template + answer_choices: Yes ||| No + id: ca363a08-c111-42d6-b643-273f2ec56249 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes reference: '' - e062b5fb-0b2d-4de3-8ebf-95a93d2eead5: !Template - answer_choices: null - id: e062b5fb-0b2d-4de3-8ebf-95a93d2eead5 + cc274ad2-c68b-4cd5-bf5e-264b533ccd29: !Template + answer_choices: 1 ||| 2 + id: cc274ad2-c68b-4cd5-bf5e-264b533ccd29 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -267,11 +283,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -279,21 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B - reference: '' - eedf81c3-95e2-4e7e-9dd1-af108079dfa9: !Template - answer_choices: Yes ||| No - id: eedf81c3-95e2-4e7e-9dd1-af108079dfa9 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' + name: grammatical_between_1_2 reference: '' diff --git a/promptsource/templates/blimp/npi_present_2/templates.yaml b/promptsource/templates/blimp/npi_present_2/templates.yaml index 06ad994bc..d402d0af9 100644 --- a/promptsource/templates/blimp/npi_present_2/templates.yaml +++ b/promptsource/templates/blimp/npi_present_2/templates.yaml @@ -1,36 +1,35 @@ dataset: blimp subset: npi_present_2 templates: - 014167ed-81cc-49dd-889c-b67b902485c0: !Template - answer_choices: null - id: 014167ed-81cc-49dd-889c-b67b902485c0 + 4d9be08d-1a50-4c43-a746-cfa4b4a40d63: !Template + answer_choices: A ||| B + id: 4d9be08d-1a50-4c43-a746-cfa4b4a40d63 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -38,14 +37,30 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: better_B_A + reference: '' + 6f4974be-561f-4cf3-b324-2e4d0c0a6c02: !Template + answer_choices: Yes ||| No + id: 6f4974be-561f-4cf3-b324-2e4d0c0a6c02 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no reference: '' - 158c8aab-7b7f-4869-8f01-32c11d450cd3: !Template - answer_choices: null - id: 158c8aab-7b7f-4869-8f01-32c11d450cd3 + 964a23ad-c863-4905-9d95-89f2f1e719aa: !Template + answer_choices: 1 ||| 2 + id: 964a23ad-c863-4905-9d95-89f2f1e719aa jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -65,11 +80,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -77,31 +92,15 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 - reference: '' - 24b330d2-525f-4f85-bf9b-3910ebe846d4: !Template - answer_choices: Yes ||| No - id: 24b330d2-525f-4f85-bf9b-3910ebe846d4 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' + name: grammatical_between_1_2 reference: '' - 4600694f-1d45-474f-a295-557c1d7189b1: !Template + a0de51d9-4521-488f-add6-8800014e8d2a: !Template answer_choices: Yes ||| No - id: 4600694f-1d45-474f-a295-557c1d7189b1 + id: a0de51d9-4521-488f-add6-8800014e8d2a jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -109,26 +108,26 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' + name: single_sentence_good_yes_no reference: '' - 812d1a83-7e31-47fb-97a4-7ac8033404ff: !Template - answer_choices: null - id: 812d1a83-7e31-47fb-97a4-7ac8033404ff + ad3bd372-17e6-49fc-acf1-0646752ef73d: !Template + answer_choices: 1 ||| 2 + id: ad3bd372-17e6-49fc-acf1-0646752ef73d jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -136,11 +135,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -148,31 +147,32 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: grammatical_which_one_1_2 reference: '' - a008869a-8ace-4ba0-a44b-c2a511030c4d: !Template + b6a4d35f-9a39-477e-be91-767599992071: !Template answer_choices: A ||| B - id: a008869a-8ace-4ba0-a44b-c2a511030c4d + id: b6a4d35f-9a39-477e-be91-767599992071 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -186,11 +186,27 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: A-B)' + name: grammatical_between_A_B + reference: '' + c3d3c859-4637-45ff-93c2-a5e28786df26: !Template + answer_choices: Yes ||| No + id: c3d3c859-4637-45ff-93c2-a5e28786df26 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes reference: '' - dcd5effe-d29d-4f21-b2eb-d2ae5148bd00: !Template + dcae6658-26fe-47d8-bd56-fa4575aa2e82: !Template answer_choices: A ||| B - id: dcd5effe-d29d-4f21-b2eb-d2ae5148bd00 + id: dcae6658-26fe-47d8-bd56-fa4575aa2e82 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -209,7 +225,7 @@ templates: {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order + Which one is a better sentence of English, A or B? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -224,27 +240,27 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: B-A)' + name: better_A_B reference: '' - e629ff75-6fe5-4408-95ee-38f234f90843: !Template + e7fed14a-3275-4240-b5f8-4db301eab377: !Template answer_choices: Yes ||| No - id: e629ff75-6fe5-4408-95ee-38f234f90843 + id: e7fed14a-3275-4240-b5f8-4db301eab377 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' + name: single_sentence_bad_no_yes reference: '' - f2062239-6ce0-49d3-92e5-8cf15fbf1bcc: !Template - answer_choices: null - id: f2062239-6ce0-49d3-92e5-8cf15fbf1bcc + f304dead-d2a1-4000-9dc7-41eb52ce4834: !Template + answer_choices: A ||| B + id: f304dead-d2a1-4000-9dc7-41eb52ce4834 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer A or B. @@ -267,11 +283,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -279,21 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B - reference: '' - fd68288e-d2a3-453b-aa68-aab00a4f3211: !Template - answer_choices: Yes ||| No - id: fd68288e-d2a3-453b-aa68-aab00a4f3211 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' + name: grammatical_which_one_A_B reference: '' diff --git a/promptsource/templates/blimp/only_npi_licensor_present/templates.yaml b/promptsource/templates/blimp/only_npi_licensor_present/templates.yaml index ed791a847..3b35e6478 100644 --- a/promptsource/templates/blimp/only_npi_licensor_present/templates.yaml +++ b/promptsource/templates/blimp/only_npi_licensor_present/templates.yaml @@ -1,9 +1,9 @@ dataset: blimp subset: only_npi_licensor_present templates: - 1cdd74f2-ab69-4c48-8617-e56e98bae3cc: !Template - answer_choices: null - id: 1cdd74f2-ab69-4c48-8617-e56e98bae3cc + 153a6b05-0890-403a-8570-4b991b48019f: !Template + answer_choices: A ||| B + id: 153a6b05-0890-403a-8570-4b991b48019f jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer A or B. @@ -26,11 +26,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -38,43 +38,49 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: grammatical_between_A_B reference: '' - 23c8d5d2-88c9-46a9-9f62-ed98c285811d: !Template - answer_choices: Yes ||| No - id: 23c8d5d2-88c9-46a9-9f62-ed98c285811d - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + 2013eb76-f3d2-407a-a267-4c993c00732a: !Template + answer_choices: A ||| B + id: 2013eb76-f3d2-407a-a267-4c993c00732a + jinja: '{% set shuffled_order = [0, 1] | random %} - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' - reference: '' - 26f64782-b7db-4ce3-98d2-5ffe7f676ebb: !Template - answer_choices: Yes ||| No - id: 26f64782-b7db-4ce3-98d2-5ffe7f676ebb - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + Read the following two sentences. - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' + original_task: true + name: better_A_B reference: '' - 352347bf-77cd-487c-b85d-35beb6be2aa6: !Template - answer_choices: null - id: 352347bf-77cd-487c-b85d-35beb6be2aa6 + 524b2e9d-9f5c-41ea-aad4-85692ce4b7f6: !Template + answer_choices: 1 ||| 2 + id: 524b2e9d-9f5c-41ea-aad4-85692ce4b7f6 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer 1 or 2. @@ -97,11 +103,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -109,42 +115,26 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: grammatical_which_one_1_2 reference: '' - 3e4e757b-d419-451d-8842-d642610ac3d6: !Template - answer_choices: Yes ||| No - id: 3e4e757b-d419-451d-8842-d642610ac3d6 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' - reference: '' - 43c9b0be-8743-4a7f-9c88-abfd06beee8a: !Template - answer_choices: null - id: 43c9b0be-8743-4a7f-9c88-abfd06beee8a + 870c9a98-31a8-423f-a030-2b6968208898: !Template + answer_choices: A ||| B + id: 870c9a98-31a8-423f-a030-2b6968208898 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -152,11 +142,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -164,11 +154,27 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: grammatical_which_one_A_B reference: '' - 6c93e5b3-38e2-454b-a56e-dd3d553ad092: !Template + ac296786-c38f-4a14-967d-0f618ea7d3cc: !Template answer_choices: Yes ||| No - id: 6c93e5b3-38e2-454b-a56e-dd3d553ad092 + id: ac296786-c38f-4a14-967d-0f618ea7d3cc + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes + reference: '' + b705748a-13d5-4e43-b827-ecfa8926b4ca: !Template + answer_choices: Yes ||| No + id: b705748a-13d5-4e43-b827-ecfa8926b4ca jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -180,11 +186,27 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' + name: single_sentence_good_yes_no reference: '' - 9bdbb8db-d390-4d2b-8503-24f08f74cf01: !Template + f0c371af-1839-448d-9dce-89fdf7388449: !Template + answer_choices: Yes ||| No + id: f0c371af-1839-448d-9dce-89fdf7388449 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes + reference: '' + f4e6761a-a597-4df7-b449-04ffb357e1c8: !Template answer_choices: A ||| B - id: 9bdbb8db-d390-4d2b-8503-24f08f74cf01 + id: f4e6761a-a597-4df7-b449-04ffb357e1c8 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -203,7 +225,7 @@ templates: {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order + Which one is a better sentence of English, B or A? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -218,70 +240,48 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: A-B)' + name: better_B_A reference: '' - c9eb9adf-5f0c-4e0b-99f3-cb8b7a508868: !Template - answer_choices: null - id: c9eb9adf-5f0c-4e0b-99f3-cb8b7a508868 - jinja: '{% set shuffled_order = [0, 1] | random %} - - Which one of the following sentences is grammatical? Please answer A or B. - - {% if shuffled_order == 0 %} - - A: {{ sentence_good }} - - B: {{ sentence_bad }} - - {% else %} - - A: {{ sentence_bad }} - - B: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - A - - {% else %} - - B + f54d7542-0a50-49d1-ab19-d839ea705642: !Template + answer_choices: Yes ||| No + id: f54d7542-0a50-49d1-ab19-d839ea705642 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - {% endif %}' + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: random_choice_A_B + original_task: false + name: single_sentence_bad_yes_no reference: '' - dc1d3756-85a7-4165-819b-c9b198b7dbb7: !Template - answer_choices: A ||| B - id: dc1d3756-85a7-4165-819b-c9b198b7dbb7 + fb88f4f5-db5b-4af8-b38b-c6ffb5eaa1ad: !Template + answer_choices: 1 ||| 2 + id: fb88f4f5-db5b-4af8-b38b-c6ffb5eaa1ad jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -295,5 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: B-A)' + name: grammatical_between_1_2 reference: '' diff --git a/promptsource/templates/blimp/only_npi_scope/templates.yaml b/promptsource/templates/blimp/only_npi_scope/templates.yaml index b781c2a2a..2435a6cb2 100644 --- a/promptsource/templates/blimp/only_npi_scope/templates.yaml +++ b/promptsource/templates/blimp/only_npi_scope/templates.yaml @@ -1,9 +1,9 @@ dataset: blimp subset: only_npi_scope templates: - 00046d74-7f92-47d6-bee8-c1ef9f705227: !Template + 05ee24e8-90c9-4dee-8f50-f5badda163ef: !Template answer_choices: A ||| B - id: 00046d74-7f92-47d6-bee8-c1ef9f705227 + id: 05ee24e8-90c9-4dee-8f50-f5badda163ef jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -22,7 +22,7 @@ templates: {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order + Which one is a better sentence of English, B or A? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -37,47 +37,32 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: A-B)' + name: better_B_A reference: '' - 1433ff94-edb2-462e-940d-333745f1287e: !Template - answer_choices: Yes ||| No - id: 1433ff94-edb2-462e-940d-333745f1287e - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' - reference: '' - 14ba38d2-20f3-48c8-8993-5713888c8ea1: !Template + 22fa63d8-54d8-47cf-b8b2-5bcdd837c3fe: !Template answer_choices: A ||| B - id: 14ba38d2-20f3-48c8-8993-5713888c8ea1 + id: 22fa63d8-54d8-47cf-b8b2-5bcdd837c3fe jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -91,14 +76,14 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: B-A)' + name: grammatical_between_A_B reference: '' - 51ad5fec-f2d0-46a2-b0f3-899464f69e42: !Template - answer_choices: null - id: 51ad5fec-f2d0-46a2-b0f3-899464f69e42 + 2e1af49d-0122-4e6f-b63c-ff97ae661aca: !Template + answer_choices: 1 ||| 2 + id: 2e1af49d-0122-4e6f-b63c-ff97ae661aca jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -118,11 +103,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -130,26 +115,26 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: grammatical_between_1_2 reference: '' - 69b54363-2a35-4285-8b32-fadeed38d419: !Template - answer_choices: null - id: 69b54363-2a35-4285-8b32-fadeed38d419 + 33504196-78bc-4f05-917d-09ad26a455d2: !Template + answer_choices: 1 ||| 2 + id: 33504196-78bc-4f05-917d-09ad26a455d2 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -157,11 +142,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -169,14 +154,62 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: grammatical_which_one_1_2 + reference: '' + 5768a12d-ad5f-430c-aa56-e9e171647568: !Template + answer_choices: Yes ||| No + id: 5768a12d-ad5f-430c-aa56-e9e171647568 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 580641d7-f207-4abc-a8d9-22bd75be999a: !Template + answer_choices: Yes ||| No + id: 580641d7-f207-4abc-a8d9-22bd75be999a + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 75ac5fed-3d82-4903-a96d-05f330418b11: !Template + answer_choices: Yes ||| No + id: 75ac5fed-3d82-4903-a96d-05f330418b11 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes reference: '' - b3dee517-bada-4cca-80e4-d7793050f5ee: !Template - answer_choices: null - id: b3dee517-bada-4cca-80e4-d7793050f5ee + e04d4f73-b27b-4ae2-802f-7a460939f37e: !Template + answer_choices: A ||| B + id: e04d4f73-b27b-4ae2-802f-7a460939f37e jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -196,11 +229,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -208,31 +241,15 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 - reference: '' - bdb12eb1-4ce4-4cc7-b9d9-646295f46d0c: !Template - answer_choices: Yes ||| No - id: bdb12eb1-4ce4-4cc7-b9d9-646295f46d0c - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' + name: grammatical_which_one_A_B reference: '' - c834463e-98c7-45d1-afe7-e9f0b61889b0: !Template + ea1796b8-c964-4040-886b-a1d10bf81f4e: !Template answer_choices: Yes ||| No - id: c834463e-98c7-45d1-afe7-e9f0b61889b0 + id: ea1796b8-c964-4040-886b-a1d10bf81f4e jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata @@ -240,54 +257,37 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' + name: single_sentence_bad_no_yes reference: '' - e8b39505-b8a5-4f3a-ba0a-ad8c60b9f173: !Template - answer_choices: Yes ||| No - id: e8b39505-b8a5-4f3a-ba0a-ad8c60b9f173 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' - reference: '' - fb0ce31d-bddb-4e3e-a4bd-b32b9f7918b1: !Template - answer_choices: null - id: fb0ce31d-bddb-4e3e-a4bd-b32b9f7918b1 + efbc668a-2116-4ad3-8cee-2cc11d7eb59e: !Template + answer_choices: A ||| B + id: efbc668a-2116-4ad3-8cee-2cc11d7eb59e jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -295,5 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: better_A_B reference: '' diff --git a/promptsource/templates/blimp/passive_1/templates.yaml b/promptsource/templates/blimp/passive_1/templates.yaml index 66427dc18..66040168c 100644 --- a/promptsource/templates/blimp/passive_1/templates.yaml +++ b/promptsource/templates/blimp/passive_1/templates.yaml @@ -1,24 +1,24 @@ dataset: blimp subset: passive_1 templates: - 05508d69-7b11-4875-bd22-b722bc0a5fed: !Template - answer_choices: null - id: 05508d69-7b11-4875-bd22-b722bc0a5fed + 7775043b-3766-49b0-8666-6a3e6fb5c1d7: !Template + answer_choices: A ||| B + id: 7775043b-3766-49b0-8666-6a3e6fb5c1d7 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -26,11 +26,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -38,27 +38,49 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: grammatical_between_A_B reference: '' - 099435ac-4216-4c5a-8c5c-47276c69ce2c: !Template - answer_choices: Yes ||| No - id: 099435ac-4216-4c5a-8c5c-47276c69ce2c - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + 7d830b2b-2599-4770-9a59-ccf0138042e1: !Template + answer_choices: A ||| B + id: 7d830b2b-2599-4770-9a59-ccf0138042e1 + jinja: '{% set shuffled_order = [0, 1] | random %} - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' + original_task: true + name: better_B_A reference: '' - 1a36c01f-575e-4226-88fb-e7c3deca5ea2: !Template + 85461fa8-c978-4ed4-860a-c72ab86f3711: !Template answer_choices: Yes ||| No - id: 1a36c01f-575e-4226-88fb-e7c3deca5ea2 + id: 85461fa8-c978-4ed4-860a-c72ab86f3711 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -70,11 +92,11 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' + name: single_sentence_good_yes_no reference: '' - 22941fb5-4e23-40e9-86df-8280bd2ca74f: !Template + 90cd6e56-369d-4d07-93b0-dc53d2bd87fd: !Template answer_choices: Yes ||| No - id: 22941fb5-4e23-40e9-86df-8280bd2ca74f + id: 90cd6e56-369d-4d07-93b0-dc53d2bd87fd jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -86,11 +108,11 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' + name: single_sentence_good_no_yes reference: '' - 5e6bee37-3698-4532-9201-7a766168a068: !Template + 9882eac8-a9c9-4302-bde6-f3beaa825fcf: !Template answer_choices: A ||| B - id: 5e6bee37-3698-4532-9201-7a766168a068 + id: 9882eac8-a9c9-4302-bde6-f3beaa825fcf jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -124,31 +146,32 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: A-B)' + name: better_A_B reference: '' - 859d74d6-b089-4cbd-a3bf-468bac5e49f2: !Template - answer_choices: A ||| B - id: 859d74d6-b089-4cbd-a3bf-468bac5e49f2 + 9caed5d3-2580-4506-91de-716d1fc35cff: !Template + answer_choices: 1 ||| 2 + id: 9caed5d3-2580-4506-91de-716d1fc35cff jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -162,53 +185,30 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: B-A)' + name: grammatical_between_1_2 reference: '' - 90d95bbb-4370-42b6-b331-8153ddf0b629: !Template - answer_choices: null - id: 90d95bbb-4370-42b6-b331-8153ddf0b629 - jinja: '{% set shuffled_order = [0, 1] | random %} - - Between the two sentences, which one is grammatical? Please answer A or B. - - {% if shuffled_order == 0 %} - - A: {{ sentence_good }} - - B: {{ sentence_bad }} - - {% else %} - - A: {{ sentence_bad }} - - B: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - A - - {% else %} - - B + b25be334-ab34-4074-b790-43247c086faf: !Template + answer_choices: Yes ||| No + id: b25be334-ab34-4074-b790-43247c086faf + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - {% endif %}' + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: random_choice_A_B_2 + original_task: false + name: single_sentence_bad_yes_no reference: '' - 9c0e4e8f-aa33-4050-85f1-a20216484922: !Template - answer_choices: null - id: 9c0e4e8f-aa33-4050-85f1-a20216484922 + caedb3f8-c46a-4820-9f34-7ea7701e2777: !Template + answer_choices: 1 ||| 2 + id: caedb3f8-c46a-4820-9f34-7ea7701e2777 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -228,11 +228,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -240,27 +240,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 - reference: '' - bcfb3886-95fa-4bbc-9b77-20fe4a3002bb: !Template - answer_choices: Yes ||| No - id: bcfb3886-95fa-4bbc-9b77-20fe4a3002bb - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' + name: grammatical_which_one_1_2 reference: '' - e264fa32-baa8-49fe-8ea4-2ec2d028768d: !Template - answer_choices: null - id: e264fa32-baa8-49fe-8ea4-2ec2d028768d + e83d5379-0a3a-4f2a-90a5-ea433e1fe667: !Template + answer_choices: A ||| B + id: e83d5379-0a3a-4f2a-90a5-ea433e1fe667 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer A or B. @@ -283,11 +267,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -295,5 +279,21 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: grammatical_which_one_A_B + reference: '' + f7a81173-2aee-460e-9205-85208d661bbd: !Template + answer_choices: Yes ||| No + id: f7a81173-2aee-460e-9205-85208d661bbd + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes reference: '' diff --git a/promptsource/templates/blimp/passive_2/templates.yaml b/promptsource/templates/blimp/passive_2/templates.yaml index ef8d2cd67..31c23eb25 100644 --- a/promptsource/templates/blimp/passive_2/templates.yaml +++ b/promptsource/templates/blimp/passive_2/templates.yaml @@ -1,45 +1,30 @@ dataset: blimp subset: passive_2 templates: - 06850205-9e5d-418a-8669-c7458c270e4a: !Template - answer_choices: Yes ||| No - id: 06850205-9e5d-418a-8669-c7458c270e4a - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' - reference: '' - 1396a0a6-c44c-4199-bc35-0a0ccb0b93dd: !Template + 024782db-3e00-4f41-bea0-2ce72772245b: !Template answer_choices: A ||| B - id: 1396a0a6-c44c-4199-bc35-0a0ccb0b93dd + id: 024782db-3e00-4f41-bea0-2ce72772245b jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -53,15 +38,15 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: B-A)' + name: grammatical_between_A_B reference: '' - 235a154a-caef-4baa-92cb-074450405264: !Template + 0c2989e5-bd53-4e90-883d-065e752a4ca9: !Template answer_choices: Yes ||| No - id: 235a154a-caef-4baa-92cb-074450405264 + id: 0c2989e5-bd53-4e90-883d-065e752a4ca9 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata @@ -69,58 +54,58 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' + name: single_sentence_bad_no_yes reference: '' - 276507d3-82cb-4cdc-9991-ff8ceeb47af2: !Template + 14a71122-0ed3-4f5d-be71-9317668e415e: !Template answer_choices: Yes ||| No - id: 276507d3-82cb-4cdc-9991-ff8ceeb47af2 + id: 14a71122-0ed3-4f5d-be71-9317668e415e jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' + name: single_sentence_good_yes_no reference: '' - 362fd9f3-e565-4145-831c-8abb42a2253f: !Template + 1e7bd836-30ce-4177-869b-67d097252829: !Template answer_choices: Yes ||| No - id: 362fd9f3-e565-4145-831c-8abb42a2253f + id: 1e7bd836-30ce-4177-869b-67d097252829 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' + name: single_sentence_bad_yes_no reference: '' - 49a67e35-5c92-43fc-b94a-bee2b1b5af29: !Template - answer_choices: null - id: 49a67e35-5c92-43fc-b94a-bee2b1b5af29 + 4422bec7-971f-4e38-af19-354ba9a225d8: !Template + answer_choices: 1 ||| 2 + id: 4422bec7-971f-4e38-af19-354ba9a225d8 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -128,11 +113,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -140,14 +125,14 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: grammatical_between_1_2 reference: '' - 87c52961-6c07-4499-965c-ec592a9de057: !Template - answer_choices: null - id: 87c52961-6c07-4499-965c-ec592a9de057 + 7ae6df52-8225-4af7-acec-920fc523d0f1: !Template + answer_choices: 1 ||| 2 + id: 7ae6df52-8225-4af7-acec-920fc523d0f1 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -167,11 +152,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -179,38 +164,37 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: grammatical_which_one_1_2 reference: '' - a31d9888-28d2-43f3-b781-033c28d16025: !Template - answer_choices: null - id: a31d9888-28d2-43f3-b781-033c28d16025 + 9e42bfca-33ae-4525-9c3a-6ee2f14e1884: !Template + answer_choices: A ||| B + id: 9e42bfca-33ae-4525-9c3a-6ee2f14e1884 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -218,38 +202,53 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: better_B_A + reference: '' + a8f3e69d-0c1e-4828-bcb2-9bced3339bab: !Template + answer_choices: Yes ||| No + id: a8f3e69d-0c1e-4828-bcb2-9bced3339bab + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes reference: '' - ade2a82f-aab8-4b21-9975-22dc3cfde035: !Template - answer_choices: null - id: ade2a82f-aab8-4b21-9975-22dc3cfde035 + b3933281-0de4-44d0-9975-68311a6ab796: !Template + answer_choices: A ||| B + id: b3933281-0de4-44d0-9975-68311a6ab796 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -257,31 +256,32 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: better_A_B reference: '' - f8abe76f-8b7f-4fb8-b8b3-3d6e524ec006: !Template + b942d542-d6cd-4105-8630-f6d5255855b8: !Template answer_choices: A ||| B - id: f8abe76f-8b7f-4fb8-b8b3-3d6e524ec006 + id: b942d542-d6cd-4105-8630-f6d5255855b8 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -295,5 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: A-B)' + name: grammatical_which_one_A_B reference: '' diff --git a/promptsource/templates/blimp/principle_A_c_command/templates.yaml b/promptsource/templates/blimp/principle_A_c_command/templates.yaml index bdfffd8f7..296b9c13c 100644 --- a/promptsource/templates/blimp/principle_A_c_command/templates.yaml +++ b/promptsource/templates/blimp/principle_A_c_command/templates.yaml @@ -1,9 +1,9 @@ dataset: blimp subset: principle_A_c_command templates: - 3a8a234f-6810-4066-85e9-0719796eecca: !Template + 28c50f2c-7001-403e-ba55-baab00d9c6b1: !Template answer_choices: Yes ||| No - id: 3a8a234f-6810-4066-85e9-0719796eecca + id: 28c50f2c-7001-403e-ba55-baab00d9c6b1 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -15,14 +15,30 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' + name: single_sentence_bad_no_yes reference: '' - 41e0a368-eb19-436c-87b3-526b5c97fd62: !Template - answer_choices: null - id: 41e0a368-eb19-436c-87b3-526b5c97fd62 + 38f2a3d9-a0f0-4317-a59a-a8fb1a9823c7: !Template + answer_choices: Yes ||| No + id: 38f2a3d9-a0f0-4317-a59a-a8fb1a9823c7 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 3a9a578f-5e96-428d-8dea-9a8098291083: !Template + answer_choices: 1 ||| 2 + id: 3a9a578f-5e96-428d-8dea-9a8098291083 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -42,11 +58,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -54,31 +70,48 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: grammatical_between_1_2 reference: '' - 45a8509c-86c2-4f82-8bc1-bb029b23e40e: !Template - answer_choices: A ||| B - id: 45a8509c-86c2-4f82-8bc1-bb029b23e40e + 53b5d892-53ab-4205-a47b-88fc01444764: !Template + answer_choices: Yes ||| No + id: 53b5d892-53ab-4205-a47b-88fc01444764 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 72be970a-653e-44c5-beef-e445239964fa: !Template + answer_choices: 1 ||| 2 + id: 72be970a-653e-44c5-beef-e445239964fa jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -92,26 +125,26 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: B-A)' + name: grammatical_which_one_1_2 reference: '' - 7a67aeb5-50b3-49cc-b3a8-e149135e9b55: !Template - answer_choices: null - id: 7a67aeb5-50b3-49cc-b3a8-e149135e9b55 + 9018b162-1d54-4a77-b5bf-552263b20447: !Template + answer_choices: A ||| B + id: 9018b162-1d54-4a77-b5bf-552263b20447 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -119,11 +152,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -131,27 +164,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 - reference: '' - 7cf054e0-5fe2-4ff6-ab66-17d8b72ece00: !Template - answer_choices: Yes ||| No - id: 7cf054e0-5fe2-4ff6-ab66-17d8b72ece00 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' + name: grammatical_between_A_B reference: '' - 87a1a3e4-753f-429c-9712-09eee687492a: !Template + 95a67aa0-f823-4a92-a3ab-cf4cc4f724e3: !Template answer_choices: A ||| B - id: 87a1a3e4-753f-429c-9712-09eee687492a + id: 95a67aa0-f823-4a92-a3ab-cf4cc4f724e3 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -170,7 +187,7 @@ templates: {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order + Which one is a better sentence of English, B or A? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -185,54 +202,37 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: A-B)' - reference: '' - 87a371fe-e4a4-43a1-a2ee-0d6d8756c450: !Template - answer_choices: Yes ||| No - id: 87a371fe-e4a4-43a1-a2ee-0d6d8756c450 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' + name: better_B_A reference: '' - 8aab460d-3ae1-4d12-afb7-121ad025f65f: !Template - answer_choices: null - id: 8aab460d-3ae1-4d12-afb7-121ad025f65f + 96442dc1-11da-4acb-9c13-99239aac99d8: !Template + answer_choices: A ||| B + id: 96442dc1-11da-4acb-9c13-99239aac99d8 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -240,30 +240,14 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: better_A_B reference: '' - df9d8964-6158-4fe7-828a-15249921878c: !Template - answer_choices: Yes ||| No - id: df9d8964-6158-4fe7-828a-15249921878c - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' - reference: '' - e6b34bec-c16a-471a-975e-ebfcda39f997: !Template - answer_choices: null - id: e6b34bec-c16a-471a-975e-ebfcda39f997 + ae57ebcd-d036-4486-9920-82c84e99702e: !Template + answer_choices: A ||| B + id: ae57ebcd-d036-4486-9920-82c84e99702e jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -283,11 +267,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -295,5 +279,21 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: grammatical_which_one_A_B + reference: '' + f4decd0e-8077-49d0-95db-65ac8cbee793: !Template + answer_choices: Yes ||| No + id: f4decd0e-8077-49d0-95db-65ac8cbee793 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes reference: '' diff --git a/promptsource/templates/blimp/principle_A_case_1/templates.yaml b/promptsource/templates/blimp/principle_A_case_1/templates.yaml index 6a4086f4f..50ae16dc3 100644 --- a/promptsource/templates/blimp/principle_A_case_1/templates.yaml +++ b/promptsource/templates/blimp/principle_A_case_1/templates.yaml @@ -1,24 +1,24 @@ dataset: blimp subset: principle_A_case_1 templates: - 006e75f2-cc80-4859-b1ae-11e44c9943d3: !Template - answer_choices: null - id: 006e75f2-cc80-4859-b1ae-11e44c9943d3 + 037c9434-fc17-4122-97f8-0cbae4474e3b: !Template + answer_choices: A ||| B + id: 037c9434-fc17-4122-97f8-0cbae4474e3b jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -26,11 +26,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -38,47 +38,32 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: grammatical_between_A_B reference: '' - 3a0dfa3b-d275-48dd-9ae6-781135300ec2: !Template - answer_choices: Yes ||| No - id: 3a0dfa3b-d275-48dd-9ae6-781135300ec2 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' - reference: '' - 53dc1cdd-da59-4e23-bfd6-87482f76c20d: !Template - answer_choices: A ||| B - id: 53dc1cdd-da59-4e23-bfd6-87482f76c20d + 053b6d4b-7920-4d03-aec0-caf6af2ae144: !Template + answer_choices: 1 ||| 2 + id: 053b6d4b-7920-4d03-aec0-caf6af2ae144 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -92,42 +77,42 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: B-A)' + name: grammatical_which_one_1_2 reference: '' - 699f6634-2536-40c5-94a8-385697c32063: !Template + 4860ae64-04b7-4e87-be92-1d8646dc0944: !Template answer_choices: Yes ||| No - id: 699f6634-2536-40c5-94a8-385697c32063 + id: 4860ae64-04b7-4e87-be92-1d8646dc0944 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' + name: single_sentence_good_no_yes reference: '' - 71efac31-53cf-4a96-9922-5d36138fe6de: !Template - answer_choices: null - id: 71efac31-53cf-4a96-9922-5d36138fe6de + 571c2f09-4dcd-496c-ae40-aaf9528124dd: !Template + answer_choices: 1 ||| 2 + id: 571c2f09-4dcd-496c-ae40-aaf9528124dd jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -135,11 +120,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -147,11 +132,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: grammatical_between_1_2 reference: '' - afcc2ba1-4b8d-4718-bfc3-6a442b4d319a: !Template + 601034e7-057a-44f6-91dc-c7fc1f13a1f9: !Template answer_choices: A ||| B - id: afcc2ba1-4b8d-4718-bfc3-6a442b4d319a + id: 601034e7-057a-44f6-91dc-c7fc1f13a1f9 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -185,46 +170,30 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: A-B)' - reference: '' - b7b20894-3825-497f-b21c-9de10513a64b: !Template - answer_choices: Yes ||| No - id: b7b20894-3825-497f-b21c-9de10513a64b - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' + name: better_A_B reference: '' - e023d42b-d5a9-42ae-9153-3fa06a8a32a9: !Template + 81a47751-1343-4e10-99c0-ad60337d0520: !Template answer_choices: Yes ||| No - id: e023d42b-d5a9-42ae-9153-3fa06a8a32a9 + id: 81a47751-1343-4e10-99c0-ad60337d0520 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' + name: single_sentence_bad_no_yes reference: '' - ecab2028-3d20-4174-be3c-ef6ba6ea813c: !Template - answer_choices: null - id: ecab2028-3d20-4174-be3c-ef6ba6ea813c + 9c059288-83e0-4cc1-b360-8dda049fbc0e: !Template + answer_choices: A ||| B + id: 9c059288-83e0-4cc1-b360-8dda049fbc0e jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -244,11 +213,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -256,38 +225,53 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: grammatical_which_one_A_B + reference: '' + b6ddaee0-a4dd-4329-8182-720025ccef2d: !Template + answer_choices: Yes ||| No + id: b6ddaee0-a4dd-4329-8182-720025ccef2d + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no reference: '' - f1ccd522-fa41-446c-b60e-372364b44274: !Template - answer_choices: null - id: f1ccd522-fa41-446c-b60e-372364b44274 + e04accc3-6f39-4446-861c-b7d2e87b0bfc: !Template + answer_choices: A ||| B + id: e04accc3-6f39-4446-861c-b7d2e87b0bfc jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -295,5 +279,21 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: better_B_A + reference: '' + eb8a8935-0a2a-499d-a030-c1add8f0e5f9: !Template + answer_choices: Yes ||| No + id: eb8a8935-0a2a-499d-a030-c1add8f0e5f9 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no reference: '' diff --git a/promptsource/templates/blimp/principle_A_case_2/templates.yaml b/promptsource/templates/blimp/principle_A_case_2/templates.yaml index 2c868a1b1..8d0d64f19 100644 --- a/promptsource/templates/blimp/principle_A_case_2/templates.yaml +++ b/promptsource/templates/blimp/principle_A_case_2/templates.yaml @@ -1,29 +1,30 @@ dataset: blimp subset: principle_A_case_2 templates: - 22577b88-67fd-4fa3-a89a-e5bb1da31047: !Template + 0355f72c-184e-4d7c-97b3-643fac97d800: !Template answer_choices: A ||| B - id: 22577b88-67fd-4fa3-a89a-e5bb1da31047 + id: 0355f72c-184e-4d7c-97b3-643fac97d800 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -37,27 +38,11 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: A-B)' - reference: '' - 5a8038d7-d922-4cf1-9447-65f732c81ccf: !Template - answer_choices: Yes ||| No - id: 5a8038d7-d922-4cf1-9447-65f732c81ccf - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' + name: grammatical_which_one_A_B reference: '' - 633080ad-aa2c-409a-8912-edf3fd02f81f: !Template - answer_choices: null - id: 633080ad-aa2c-409a-8912-edf3fd02f81f + 1b8c4d7c-e7ae-4053-9dd1-eafc41b509c0: !Template + answer_choices: 1 ||| 2 + id: 1b8c4d7c-e7ae-4053-9dd1-eafc41b509c0 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -80,11 +65,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -92,15 +77,15 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: grammatical_between_1_2 reference: '' - 80231c1a-b0a4-4d6c-88b4-b81d708c8442: !Template + 3009bf8b-81ac-4cd2-b421-4ad701735004: !Template answer_choices: Yes ||| No - id: 80231c1a-b0a4-4d6c-88b4-b81d708c8442 + id: 3009bf8b-81ac-4cd2-b421-4ad701735004 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata @@ -108,26 +93,64 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' + name: single_sentence_bad_yes_no reference: '' - 8a1b6b9a-20f1-4450-a848-3838c45fa7e3: !Template - answer_choices: null - id: 8a1b6b9a-20f1-4450-a848-3838c45fa7e3 + 38abf4cd-fb02-40a8-8c86-8b3f0d5aa007: !Template + answer_choices: A ||| B + id: 38abf4cd-fb02-40a8-8c86-8b3f0d5aa007 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: better_A_B + reference: '' + 58866b5c-0625-487f-b859-0b968c29f745: !Template + answer_choices: A ||| B + id: 58866b5c-0625-487f-b859-0b968c29f745 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} {% endif %} @@ -135,11 +158,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -147,11 +170,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: grammatical_between_A_B reference: '' - 9f8b70a6-28ac-4ee7-adf4-6a0d99ab7f61: !Template + 6d0da650-5594-4f61-9e61-112205cdad64: !Template answer_choices: Yes ||| No - id: 9f8b70a6-28ac-4ee7-adf4-6a0d99ab7f61 + id: 6d0da650-5594-4f61-9e61-112205cdad64 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -163,11 +186,11 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' + name: single_sentence_good_yes_no reference: '' - afbc2fa9-0571-4d74-a9a2-a029650c604a: !Template + a5e376d0-4d84-4763-8a22-13995232fb4d: !Template answer_choices: Yes ||| No - id: afbc2fa9-0571-4d74-a9a2-a029650c604a + id: a5e376d0-4d84-4763-8a22-13995232fb4d jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -179,26 +202,26 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' + name: single_sentence_good_no_yes reference: '' - bd7eb31a-0b27-48b5-a97e-a060c93c57bd: !Template - answer_choices: null - id: bd7eb31a-0b27-48b5-a97e-a060c93c57bd + b149dd7a-1f6f-43db-beb0-0d926149576b: !Template + answer_choices: 1 ||| 2 + id: b149dd7a-1f6f-43db-beb0-0d926149576b jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -206,11 +229,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -218,11 +241,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: grammatical_which_one_1_2 reference: '' - d2df2724-697e-453a-9a9d-d3d149bab653: !Template + bf408753-f905-40f8-9c58-9601f98ce7d7: !Template answer_choices: A ||| B - id: d2df2724-697e-453a-9a9d-d3d149bab653 + id: bf408753-f905-40f8-9c58-9601f98ce7d7 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -256,44 +279,21 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: B-A)' + name: better_B_A reference: '' - d497385a-b15a-4677-a92b-bd0c9f2e4434: !Template - answer_choices: null - id: d497385a-b15a-4677-a92b-bd0c9f2e4434 - jinja: '{% set shuffled_order = [0, 1] | random %} - - Between the two sentences, which one is grammatical? Please answer A or B. - - {% if shuffled_order == 0 %} - - A: {{ sentence_good }} - - B: {{ sentence_bad }} - - {% else %} - - A: {{ sentence_bad }} - - B: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - A - - {% else %} - - B + c52c20cc-369d-4d77-ad55-605f1d046255: !Template + answer_choices: Yes ||| No + id: c52c20cc-369d-4d77-ad55-605f1d046255 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - {% endif %}' + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: random_choice_A_B_2 + original_task: false + name: single_sentence_bad_no_yes reference: '' diff --git a/promptsource/templates/blimp/principle_A_domain_1/templates.yaml b/promptsource/templates/blimp/principle_A_domain_1/templates.yaml index a840e53b1..588865d41 100644 --- a/promptsource/templates/blimp/principle_A_domain_1/templates.yaml +++ b/promptsource/templates/blimp/principle_A_domain_1/templates.yaml @@ -1,28 +1,12 @@ dataset: blimp subset: principle_A_domain_1 templates: - 19cb8530-a493-4f6e-afd4-96658e47480f: !Template - answer_choices: Yes ||| No - id: 19cb8530-a493-4f6e-afd4-96658e47480f - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' - reference: '' - 83d5c36b-4acd-4db9-9bd1-aa8954c571fa: !Template - answer_choices: null - id: 83d5c36b-4acd-4db9-9bd1-aa8954c571fa + 196cf1b1-28b6-4f4d-9e7c-51c7e2bbbd88: !Template + answer_choices: A ||| B + id: 196cf1b1-28b6-4f4d-9e7c-51c7e2bbbd88 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -42,11 +26,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -54,27 +38,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 - reference: '' - 850265b6-8463-47dc-b019-a01a033fd6d6: !Template - answer_choices: Yes ||| No - id: 850265b6-8463-47dc-b019-a01a033fd6d6 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' + name: grammatical_which_one_A_B reference: '' - 96b0ff1d-ba98-4ccb-8c27-472d93e06eb9: !Template + 1c73db3f-859a-4a0a-b636-7ef0693ccb7e: !Template answer_choices: A ||| B - id: 96b0ff1d-ba98-4ccb-8c27-472d93e06eb9 + id: 1c73db3f-859a-4a0a-b636-7ef0693ccb7e jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -93,7 +61,7 @@ templates: {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order + Which one is a better sentence of English, A or B? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -108,26 +76,42 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: B-A)' + name: better_A_B reference: '' - 97939c02-5c5f-40b0-b22c-0fbc430c5730: !Template - answer_choices: null - id: 97939c02-5c5f-40b0-b22c-0fbc430c5730 + 1d2a8265-0ea0-449d-bd83-b0f93650eafb: !Template + answer_choices: Yes ||| No + id: 1d2a8265-0ea0-449d-bd83-b0f93650eafb + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes + reference: '' + 49bb9a16-e2dc-4ba2-81a3-3c47b5525f4d: !Template + answer_choices: 1 ||| 2 + id: 49bb9a16-e2dc-4ba2-81a3-3c47b5525f4d jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -135,11 +119,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -147,43 +131,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B - reference: '' - be1cdccf-c158-49a5-bdc0-c34d5973eafe: !Template - answer_choices: Yes ||| No - id: be1cdccf-c158-49a5-bdc0-c34d5973eafe - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' - reference: '' - e0518dfb-31b3-4bb4-953d-877ae9394b6e: !Template - answer_choices: Yes ||| No - id: e0518dfb-31b3-4bb4-953d-877ae9394b6e - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' + name: grammatical_between_1_2 reference: '' - ed330c7f-4450-4abb-9344-56e7a53d610a: !Template - answer_choices: null - id: ed330c7f-4450-4abb-9344-56e7a53d610a + 5f6c71f1-f5c3-45bc-bc8d-79e79c85a346: !Template + answer_choices: 1 ||| 2 + id: 5f6c71f1-f5c3-45bc-bc8d-79e79c85a346 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer 1 or 2. @@ -206,11 +158,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -218,38 +170,37 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: grammatical_which_one_1_2 reference: '' - f1f5e14e-d56b-41b2-9ab8-99228e95e213: !Template - answer_choices: null - id: f1f5e14e-d56b-41b2-9ab8-99228e95e213 + 67e21963-a1e8-4e0c-b9f4-1a98e610c85f: !Template + answer_choices: A ||| B + id: 67e21963-a1e8-4e0c-b9f4-1a98e610c85f jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -257,31 +208,80 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: better_B_A + reference: '' + 6bf676f9-4dfe-478d-b041-e31370e68576: !Template + answer_choices: Yes ||| No + id: 6bf676f9-4dfe-478d-b041-e31370e68576 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes reference: '' - fa68b1c8-becf-49e3-a504-e3f6acc9a12e: !Template + 6c8e382e-6614-4d86-b042-764fd98c60b5: !Template + answer_choices: Yes ||| No + id: 6c8e382e-6614-4d86-b042-764fd98c60b5 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + af9fd8e7-9bdb-403b-8a8c-3015d644e410: !Template + answer_choices: Yes ||| No + id: af9fd8e7-9bdb-403b-8a8c-3015d644e410 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + bd9fc57d-129f-4ad3-a266-adbb6558201d: !Template answer_choices: A ||| B - id: fa68b1c8-becf-49e3-a504-e3f6acc9a12e + id: bd9fc57d-129f-4ad3-a266-adbb6558201d jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -295,5 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: A-B)' + name: grammatical_between_A_B reference: '' diff --git a/promptsource/templates/blimp/principle_A_domain_2/templates.yaml b/promptsource/templates/blimp/principle_A_domain_2/templates.yaml index 2f6c2f7be..080be694d 100644 --- a/promptsource/templates/blimp/principle_A_domain_2/templates.yaml +++ b/promptsource/templates/blimp/principle_A_domain_2/templates.yaml @@ -1,45 +1,30 @@ dataset: blimp subset: principle_A_domain_2 templates: - 1f88e383-0be4-48e8-918b-0fc3211c27c5: !Template - answer_choices: Yes ||| No - id: 1f88e383-0be4-48e8-918b-0fc3211c27c5 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' - reference: '' - 22ac623e-f682-430b-a8ce-bf61b0ccbc03: !Template + 1bc7e35a-0ef1-49df-b0e0-21a60b7620ef: !Template answer_choices: A ||| B - id: 22ac623e-f682-430b-a8ce-bf61b0ccbc03 + id: 1bc7e35a-0ef1-49df-b0e0-21a60b7620ef jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -53,14 +38,14 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: B-A)' + name: grammatical_which_one_A_B reference: '' - 3a9794d2-8772-4a62-bb55-ea1f80ccdabb: !Template - answer_choices: null - id: 3a9794d2-8772-4a62-bb55-ea1f80ccdabb + 34a9cc3d-6c52-4c4e-96f2-033ddb285652: !Template + answer_choices: A ||| B + id: 34a9cc3d-6c52-4c4e-96f2-033ddb285652 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -80,11 +65,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -92,38 +77,53 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: grammatical_between_A_B reference: '' - 41bc5343-54e9-4ae2-949d-810f5702c903: !Template - answer_choices: null - id: 41bc5343-54e9-4ae2-949d-810f5702c903 + 43259012-399d-4838-96bd-d3e9824baf29: !Template + answer_choices: Yes ||| No + id: 43259012-399d-4838-96bd-d3e9824baf29 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes + reference: '' + 46bbba41-aa08-447e-901b-f1de07ec3fff: !Template + answer_choices: A ||| B + id: 46bbba41-aa08-447e-901b-f1de07ec3fff jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -131,38 +131,53 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: better_B_A + reference: '' + 719373b4-beba-4c4d-ad14-8d77c1e2b9e8: !Template + answer_choices: Yes ||| No + id: 719373b4-beba-4c4d-ad14-8d77c1e2b9e8 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no reference: '' - a6903eb8-a4a3-479d-83cf-34c69f35bde2: !Template - answer_choices: null - id: a6903eb8-a4a3-479d-83cf-34c69f35bde2 + 75c53e28-0b7c-40cf-bac6-0b06c3a881bd: !Template + answer_choices: A ||| B + id: 75c53e28-0b7c-40cf-bac6-0b06c3a881bd jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -170,14 +185,14 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: better_A_B reference: '' - e096e078-8553-4183-8cfd-ae1b15dce783: !Template - answer_choices: null - id: e096e078-8553-4183-8cfd-ae1b15dce783 + 89c6f7da-cf9e-4deb-b11a-67943c938d78: !Template + answer_choices: 1 ||| 2 + id: 89c6f7da-cf9e-4deb-b11a-67943c938d78 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -197,11 +212,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -209,47 +224,48 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: grammatical_which_one_1_2 reference: '' - e43587f3-9cd6-4c70-935b-5453e7620967: !Template + a2ff9305-d52e-4506-a2fc-4708e8d66621: !Template answer_choices: Yes ||| No - id: e43587f3-9cd6-4c70-935b-5453e7620967 + id: a2ff9305-d52e-4506-a2fc-4708e8d66621 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' + name: single_sentence_bad_no_yes reference: '' - edfcc4b1-5de9-4d28-8420-5dfbe8dff874: !Template - answer_choices: A ||| B - id: edfcc4b1-5de9-4d28-8420-5dfbe8dff874 + e66c21ed-fd59-45ee-8515-a8e729a3f352: !Template + answer_choices: 1 ||| 2 + id: e66c21ed-fd59-45ee-8515-a8e729a3f352 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -263,31 +279,15 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: A-B)' - reference: '' - efde4ccd-524a-435c-a037-4e9fcc938e5a: !Template - answer_choices: Yes ||| No - id: efde4ccd-524a-435c-a037-4e9fcc938e5a - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' + name: grammatical_between_1_2 reference: '' - f30a0040-03da-4750-9a37-16ed02ee1fa2: !Template + ec388750-a9cc-49a2-8cc1-1dd619ef8b01: !Template answer_choices: Yes ||| No - id: f30a0040-03da-4750-9a37-16ed02ee1fa2 + id: ec388750-a9cc-49a2-8cc1-1dd619ef8b01 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata @@ -295,5 +295,5 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' + name: single_sentence_bad_yes_no reference: '' diff --git a/promptsource/templates/blimp/principle_A_domain_3/templates.yaml b/promptsource/templates/blimp/principle_A_domain_3/templates.yaml index f20f307ba..2c8cb11ee 100644 --- a/promptsource/templates/blimp/principle_A_domain_3/templates.yaml +++ b/promptsource/templates/blimp/principle_A_domain_3/templates.yaml @@ -1,12 +1,12 @@ dataset: blimp subset: principle_A_domain_3 templates: - 25577715-21c2-482c-b8a6-8e4e14af5dd2: !Template - answer_choices: null - id: 25577715-21c2-482c-b8a6-8e4e14af5dd2 + 08f945b2-0396-4b0e-a7e7-32678c2f4580: !Template + answer_choices: 1 ||| 2 + id: 08f945b2-0396-4b0e-a7e7-32678c2f4580 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -26,11 +26,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -38,15 +38,15 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: grammatical_between_1_2 reference: '' - 4a98eef9-e316-44f0-b8e5-ecc7d7adcf54: !Template + 0a205494-c642-4158-af18-9e9693a1bfdc: !Template answer_choices: Yes ||| No - id: 4a98eef9-e316-44f0-b8e5-ecc7d7adcf54 + id: 0a205494-c642-4158-af18-9e9693a1bfdc jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata @@ -54,11 +54,50 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' + name: single_sentence_bad_no_yes + reference: '' + 3b34c4b3-c1e2-4300-bb4f-ecdb4827aff9: !Template + answer_choices: A ||| B + id: 3b34c4b3-c1e2-4300-bb4f-ecdb4827aff9 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_A_B reference: '' - 60028ba5-b56b-406b-8f78-5a8b38a1f885: !Template + 3b54e84f-ee8f-4331-b127-6db5144f9b5a: !Template answer_choices: Yes ||| No - id: 60028ba5-b56b-406b-8f78-5a8b38a1f885 + id: 3b54e84f-ee8f-4331-b127-6db5144f9b5a jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -70,11 +109,27 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' + name: single_sentence_good_no_yes + reference: '' + 5802d66e-5745-4ab7-99e9-20a55bac70d4: !Template + answer_choices: Yes ||| No + id: 5802d66e-5745-4ab7-99e9-20a55bac70d4 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no reference: '' - 658a5728-498d-4aac-8274-627a72a83d80: !Template + 741102c7-568c-4cf4-b4d4-83689c1af124: !Template answer_choices: A ||| B - id: 658a5728-498d-4aac-8274-627a72a83d80 + id: 741102c7-568c-4cf4-b4d4-83689c1af124 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -108,26 +163,26 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: A-B)' + name: better_A_B reference: '' - 6f9de6aa-a2fa-4d2d-8b18-6e620b20a19b: !Template - answer_choices: null - id: 6f9de6aa-a2fa-4d2d-8b18-6e620b20a19b + 99aebe4a-8bfd-4c28-bc6b-fdb1ab2a41d6: !Template + answer_choices: 1 ||| 2 + id: 99aebe4a-8bfd-4c28-bc6b-fdb1ab2a41d6 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -135,11 +190,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -147,26 +202,26 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: grammatical_which_one_1_2 reference: '' - 71a4569a-a82f-431b-b443-a5b953a3487c: !Template - answer_choices: null - id: 71a4569a-a82f-431b-b443-a5b953a3487c + b296b798-0d53-46e9-80f1-c6c66860b38b: !Template + answer_choices: A ||| B + id: b296b798-0d53-46e9-80f1-c6c66860b38b jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -174,11 +229,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -186,15 +241,15 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: grammatical_which_one_A_B reference: '' - 9413e859-5ebe-43b9-a787-f2d2d14632ea: !Template + c906716d-27af-4ee8-b0cc-e7137f5ee43b: !Template answer_choices: Yes ||| No - id: 9413e859-5ebe-43b9-a787-f2d2d14632ea + id: c906716d-27af-4ee8-b0cc-e7137f5ee43b jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata @@ -202,27 +257,11 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' - reference: '' - a0905196-621f-47e5-82f7-15e7027accd4: !Template - answer_choices: Yes ||| No - id: a0905196-621f-47e5-82f7-15e7027accd4 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' + name: single_sentence_bad_yes_no reference: '' - c9f91569-0add-4b92-b33c-b312f700c48d: !Template + dba98473-151b-42f1-853f-db5eb8057936: !Template answer_choices: A ||| B - id: c9f91569-0add-4b92-b33c-b312f700c48d + id: dba98473-151b-42f1-853f-db5eb8057936 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -256,44 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: B-A)' - reference: '' - e885ab79-3348-427e-ab40-1714874db66c: !Template - answer_choices: null - id: e885ab79-3348-427e-ab40-1714874db66c - jinja: '{% set shuffled_order = [0, 1] | random %} - - Between the two sentences, which one is grammatical? Please answer A or B. - - {% if shuffled_order == 0 %} - - A: {{ sentence_good }} - - B: {{ sentence_bad }} - - {% else %} - - A: {{ sentence_bad }} - - B: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - A - - {% else %} - - B - - {% endif %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: random_choice_A_B_2 + name: better_B_A reference: '' diff --git a/promptsource/templates/blimp/principle_A_reconstruction/templates.yaml b/promptsource/templates/blimp/principle_A_reconstruction/templates.yaml index 8b6fba67e..60afe26c0 100644 --- a/promptsource/templates/blimp/principle_A_reconstruction/templates.yaml +++ b/promptsource/templates/blimp/principle_A_reconstruction/templates.yaml @@ -1,29 +1,62 @@ dataset: blimp subset: principle_A_reconstruction templates: - 1f826642-2a9d-471f-adf6-90d6e4eef81a: !Template + 3d9f42c6-bd69-4cb8-9bb3-44d2cc95c6cf: !Template + answer_choices: Yes ||| No + id: 3d9f42c6-bd69-4cb8-9bb3-44d2cc95c6cf + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 76b92be2-35c7-4b4b-85a1-3b71f3f9a0aa: !Template + answer_choices: Yes ||| No + id: 76b92be2-35c7-4b4b-85a1-3b71f3f9a0aa + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 92401ec1-563a-4b64-9dc7-a96529aafc9b: !Template answer_choices: A ||| B - id: 1f826642-2a9d-471f-adf6-90d6e4eef81a + id: 92401ec1-563a-4b64-9dc7-a96529aafc9b jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -37,31 +70,32 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: A-B)' + name: grammatical_which_one_A_B reference: '' - 344f1444-76d0-477f-8fcf-3960fda4ac5a: !Template - answer_choices: A ||| B - id: 344f1444-76d0-477f-8fcf-3960fda4ac5a + 9e15325b-a26b-4fbc-b92b-4ca4ec48d283: !Template + answer_choices: 1 ||| 2 + id: 9e15325b-a26b-4fbc-b92b-4ca4ec48d283 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -75,15 +109,15 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: B-A)' + name: grammatical_between_1_2 reference: '' - 364c32f7-567d-4eef-b632-42dd83f0203d: !Template + b3f82dfc-421d-470d-ba43-c21ebbfbed3c: !Template answer_choices: Yes ||| No - id: 364c32f7-567d-4eef-b632-42dd83f0203d + id: b3f82dfc-421d-470d-ba43-c21ebbfbed3c jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -91,58 +125,26 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' - reference: '' - 43bba55d-06d6-4478-81ff-b20dee650280: !Template - answer_choices: Yes ||| No - id: 43bba55d-06d6-4478-81ff-b20dee650280 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' - reference: '' - 4e92194b-15f4-472e-b41d-e006b1d256f7: !Template - answer_choices: Yes ||| No - id: 4e92194b-15f4-472e-b41d-e006b1d256f7 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' + name: single_sentence_good_no_yes reference: '' - 5d38a794-6a00-467d-b582-589ddb0b9367: !Template - answer_choices: null - id: 5d38a794-6a00-467d-b582-589ddb0b9367 + c0392d6b-4b76-439b-8683-d5a50ba97b9e: !Template + answer_choices: A ||| B + id: c0392d6b-4b76-439b-8683-d5a50ba97b9e jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -150,11 +152,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -162,38 +164,37 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: grammatical_between_A_B reference: '' - 60d55500-45c5-4d89-980d-68ba36e31819: !Template - answer_choices: null - id: 60d55500-45c5-4d89-980d-68ba36e31819 + c64d706d-53d8-4d9e-b468-7f80f8cc787b: !Template + answer_choices: A ||| B + id: c64d706d-53d8-4d9e-b468-7f80f8cc787b jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -201,38 +202,37 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: better_B_A reference: '' - bafaa33b-96ac-4bda-bffb-c3c5e113e3e9: !Template - answer_choices: null - id: bafaa33b-96ac-4bda-bffb-c3c5e113e3e9 + e1eb7e07-3be2-4c15-9e0d-b1f39dd40eb2: !Template + answer_choices: A ||| B + id: e1eb7e07-3be2-4c15-9e0d-b1f39dd40eb2 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -240,14 +240,14 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: better_A_B reference: '' - e1dc08d8-9353-4f8b-b0be-1a45c7a665d5: !Template - answer_choices: null - id: e1dc08d8-9353-4f8b-b0be-1a45c7a665d5 + e27d8725-2082-4b0a-a759-44dd4921396b: !Template + answer_choices: 1 ||| 2 + id: e27d8725-2082-4b0a-a759-44dd4921396b jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -267,11 +267,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -279,21 +279,21 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: grammatical_which_one_1_2 reference: '' - fdd1ca40-42db-4204-9a1f-f491c31cac93: !Template + e8ceafa4-e73d-476b-bbfc-abf93d867e23: !Template answer_choices: Yes ||| No - id: fdd1ca40-42db-4204-9a1f-f491c31cac93 + id: e8ceafa4-e73d-476b-bbfc-abf93d867e23 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' + name: single_sentence_bad_no_yes reference: '' diff --git a/promptsource/templates/blimp/regular_plural_subject_verb_agreement_1/templates.yaml b/promptsource/templates/blimp/regular_plural_subject_verb_agreement_1/templates.yaml index e4f843b79..aafdb829b 100644 --- a/promptsource/templates/blimp/regular_plural_subject_verb_agreement_1/templates.yaml +++ b/promptsource/templates/blimp/regular_plural_subject_verb_agreement_1/templates.yaml @@ -1,68 +1,35 @@ dataset: blimp subset: regular_plural_subject_verb_agreement_1 templates: - 036a4f23-0480-41e4-86a3-d97c633c6312: !Template - answer_choices: Yes ||| No - id: 036a4f23-0480-41e4-86a3-d97c633c6312 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' - reference: '' - 2190e8d7-6a9b-460b-99cd-a35f29ebd8cb: !Template - answer_choices: Yes ||| No - id: 2190e8d7-6a9b-460b-99cd-a35f29ebd8cb - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' - reference: '' - 235968d0-2297-4cf1-abf6-807b232c599e: !Template - answer_choices: null - id: 235968d0-2297-4cf1-abf6-807b232c599e + 022a7339-fa4f-4177-8f64-d8b232d4f8f2: !Template + answer_choices: A ||| B + id: 022a7339-fa4f-4177-8f64-d8b232d4f8f2 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -70,63 +37,32 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: better_A_B reference: '' - 2fd30e23-066b-45d5-81cf-8735775dfa6e: !Template - answer_choices: Yes ||| No - id: 2fd30e23-066b-45d5-81cf-8735775dfa6e - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' - reference: '' - 32ab26ea-16ff-4bfb-bc88-46f9028cbc34: !Template - answer_choices: Yes ||| No - id: 32ab26ea-16ff-4bfb-bc88-46f9028cbc34 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' - reference: '' - 716ca2d5-8cbd-4271-aa66-c5e2f4de41e8: !Template + 248db1ea-7f34-4b5e-98a1-990fa3f2602e: !Template answer_choices: A ||| B - id: 716ca2d5-8cbd-4271-aa66-c5e2f4de41e8 + id: 248db1ea-7f34-4b5e-98a1-990fa3f2602e jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -140,11 +76,11 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: B-A)' + name: grammatical_which_one_A_B reference: '' - 87d503e6-f4e6-489e-8b4f-87d438be543b: !Template - answer_choices: null - id: 87d503e6-f4e6-489e-8b4f-87d438be543b + 3d7a95a5-cb60-460b-b1de-bee81fcb600e: !Template + answer_choices: 1 ||| 2 + id: 3d7a95a5-cb60-460b-b1de-bee81fcb600e jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -167,11 +103,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -179,26 +115,42 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: grammatical_between_1_2 + reference: '' + 47884791-2028-460c-8c7a-71c05a606cf3: !Template + answer_choices: Yes ||| No + id: 47884791-2028-460c-8c7a-71c05a606cf3 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes reference: '' - 9e59aee4-eb66-4ab8-bf36-6c0d15e35cbd: !Template - answer_choices: null - id: 9e59aee4-eb66-4ab8-bf36-6c0d15e35cbd + 6fc714a5-6a0c-4938-af80-297c63864f8d: !Template + answer_choices: A ||| B + id: 6fc714a5-6a0c-4938-af80-297c63864f8d jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -206,11 +158,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -218,31 +170,80 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: grammatical_between_A_B reference: '' - aad80196-c1f1-4244-b8dc-49bd17507fd7: !Template - answer_choices: A ||| B - id: aad80196-c1f1-4244-b8dc-49bd17507fd7 + 7b27d408-68b0-4440-aa36-a6ad32ddec8c: !Template + answer_choices: Yes ||| No + id: 7b27d408-68b0-4440-aa36-a6ad32ddec8c + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 8a7b76ea-5b2e-4a5d-930c-8498372cca63: !Template + answer_choices: Yes ||| No + id: 8a7b76ea-5b2e-4a5d-930c-8498372cca63 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 8bcadc10-d2eb-4258-829a-90203e050487: !Template + answer_choices: Yes ||| No + id: 8bcadc10-d2eb-4258-829a-90203e050487 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes + reference: '' + ac11f4e8-8b7e-4f4e-99f9-2e0242484ce2: !Template + answer_choices: 1 ||| 2 + id: ac11f4e8-8b7e-4f4e-99f9-2e0242484ce2 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -256,38 +257,37 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: A-B)' + name: grammatical_which_one_1_2 reference: '' - fe59efd6-dfc0-4e6c-97fe-c5e6da764f5e: !Template - answer_choices: null - id: fe59efd6-dfc0-4e6c-97fe-c5e6da764f5e + b0ff3c16-d083-48bc-8074-bc9ab25caa9a: !Template + answer_choices: A ||| B + id: b0ff3c16-d083-48bc-8074-bc9ab25caa9a jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -295,5 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: better_B_A reference: '' diff --git a/promptsource/templates/blimp/regular_plural_subject_verb_agreement_2/templates.yaml b/promptsource/templates/blimp/regular_plural_subject_verb_agreement_2/templates.yaml index a7df59118..7999a0684 100644 --- a/promptsource/templates/blimp/regular_plural_subject_verb_agreement_2/templates.yaml +++ b/promptsource/templates/blimp/regular_plural_subject_verb_agreement_2/templates.yaml @@ -1,12 +1,12 @@ dataset: blimp subset: regular_plural_subject_verb_agreement_2 templates: - 14974aa7-8738-4dcd-9d8a-e3dbf879ab54: !Template - answer_choices: null - id: 14974aa7-8738-4dcd-9d8a-e3dbf879ab54 + 039cee11-ac01-4aba-8667-a001c4a66428: !Template + answer_choices: 1 ||| 2 + id: 039cee11-ac01-4aba-8667-a001c4a66428 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -26,11 +26,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -38,11 +38,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: grammatical_which_one_1_2 reference: '' - 24a9f478-f361-4cd0-a638-c500f887a582: !Template + 5c94d850-6a7a-4ed4-bc48-3a432b21f194: !Template answer_choices: A ||| B - id: 24a9f478-f361-4cd0-a638-c500f887a582 + id: 5c94d850-6a7a-4ed4-bc48-3a432b21f194 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -76,59 +76,11 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: A-B)' - reference: '' - 2ee2667d-4540-4ded-9276-ea3d9170034d: !Template - answer_choices: Yes ||| No - id: 2ee2667d-4540-4ded-9276-ea3d9170034d - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' + name: better_A_B reference: '' - 62d16043-8893-4a9b-8b2f-705015a2f7e8: !Template + 5cde73e5-5556-49e4-aed7-b0e80e38f92d: !Template answer_choices: Yes ||| No - id: 62d16043-8893-4a9b-8b2f-705015a2f7e8 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' - reference: '' - 708f47c9-847d-4b98-8564-6c2584fe63bc: !Template - answer_choices: Yes ||| No - id: 708f47c9-847d-4b98-8564-6c2584fe63bc - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' - reference: '' - 73a4f13d-6010-4a08-80dc-c5cd9a30eba3: !Template - answer_choices: Yes ||| No - id: 73a4f13d-6010-4a08-80dc-c5cd9a30eba3 + id: 5cde73e5-5556-49e4-aed7-b0e80e38f92d jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -140,31 +92,32 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' + name: single_sentence_good_no_yes reference: '' - a223dc66-85c0-4e9e-9cab-2d8d41897b09: !Template + 6586899d-4afd-425f-b4a0-eb7f1e7b4d33: !Template answer_choices: A ||| B - id: a223dc66-85c0-4e9e-9cab-2d8d41897b09 + id: 6586899d-4afd-425f-b4a0-eb7f1e7b4d33 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -178,26 +131,26 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: B-A)' + name: grammatical_between_A_B reference: '' - bfa1b9c4-68a6-4f4d-8af3-b4b2cf4cee65: !Template - answer_choices: null - id: bfa1b9c4-68a6-4f4d-8af3-b4b2cf4cee65 + bc0fd9b9-e5b3-4e66-b991-d9be54fda16b: !Template + answer_choices: 1 ||| 2 + id: bc0fd9b9-e5b3-4e66-b991-d9be54fda16b jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -205,11 +158,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -217,38 +170,37 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: grammatical_between_1_2 reference: '' - c7f3752d-fc38-46f2-8f47-223108a3e042: !Template - answer_choices: null - id: c7f3752d-fc38-46f2-8f47-223108a3e042 + dac79382-484c-4581-9075-e5691fae76b7: !Template + answer_choices: A ||| B + id: dac79382-484c-4581-9075-e5691fae76b7 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -256,11 +208,59 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: better_B_A reference: '' - caa0cd60-20e8-44c5-aeda-516a8bf649f7: !Template - answer_choices: null - id: caa0cd60-20e8-44c5-aeda-516a8bf649f7 + dfe5bb12-4134-4615-9547-2ee980c784f0: !Template + answer_choices: Yes ||| No + id: dfe5bb12-4134-4615-9547-2ee980c784f0 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes + reference: '' + e0f5744e-c31a-4ae1-aa6c-a92b82fc3a1e: !Template + answer_choices: Yes ||| No + id: e0f5744e-c31a-4ae1-aa6c-a92b82fc3a1e + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + ea7aa88c-8ad2-4cc0-8af3-ef647f7bd173: !Template + answer_choices: Yes ||| No + id: ea7aa88c-8ad2-4cc0-8af3-ef647f7bd173 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + f44231dc-4eee-4e9d-945c-70225b6e0c43: !Template + answer_choices: A ||| B + id: f44231dc-4eee-4e9d-945c-70225b6e0c43 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer A or B. @@ -283,11 +283,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -295,5 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: grammatical_which_one_A_B reference: '' diff --git a/promptsource/templates/blimp/sentential_negation_npi_licensor_present/templates.yaml b/promptsource/templates/blimp/sentential_negation_npi_licensor_present/templates.yaml index e86b8a7fe..d9450a20f 100644 --- a/promptsource/templates/blimp/sentential_negation_npi_licensor_present/templates.yaml +++ b/promptsource/templates/blimp/sentential_negation_npi_licensor_present/templates.yaml @@ -1,29 +1,30 @@ dataset: blimp subset: sentential_negation_npi_licensor_present templates: - 02730feb-390f-4f6e-91f5-742263d87ef3: !Template - answer_choices: A ||| B - id: 02730feb-390f-4f6e-91f5-742263d87ef3 + 20f8866d-81c6-4eb0-b9ae-0d0028c78e77: !Template + answer_choices: 1 ||| 2 + id: 20f8866d-81c6-4eb0-b9ae-0d0028c78e77 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -37,15 +38,15 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: A-B)' + name: grammatical_which_one_1_2 reference: '' - 2af72d68-0fe9-492e-9532-1e0033764dd6: !Template + 49cd246c-4a62-431c-b4a6-efbd75a05284: !Template answer_choices: Yes ||| No - id: 2af72d68-0fe9-492e-9532-1e0033764dd6 + id: 49cd246c-4a62-431c-b4a6-efbd75a05284 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata @@ -53,38 +54,37 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' + name: single_sentence_bad_no_yes reference: '' - 5574f54b-db6e-4dd8-ba03-6c7d365688b2: !Template - answer_choices: null - id: 5574f54b-db6e-4dd8-ba03-6c7d365688b2 + 4d2b0ac9-8c1c-4577-b5b4-018ca2e4cf7b: !Template + answer_choices: A ||| B + id: 4d2b0ac9-8c1c-4577-b5b4-018ca2e4cf7b jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -92,42 +92,26 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 - reference: '' - 6357d10b-82a3-4f91-a79f-a905542c75d4: !Template - answer_choices: Yes ||| No - id: 6357d10b-82a3-4f91-a79f-a905542c75d4 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' + name: better_B_A reference: '' - 6d18ce1d-ca55-44fd-9041-1f27bcef0609: !Template - answer_choices: null - id: 6d18ce1d-ca55-44fd-9041-1f27bcef0609 + 519cf39e-5aee-489a-90ee-bdbcb55d0e71: !Template + answer_choices: A ||| B + id: 519cf39e-5aee-489a-90ee-bdbcb55d0e71 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -135,11 +119,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -147,31 +131,32 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: grammatical_between_A_B reference: '' - 6f84156e-22ae-4c07-8e3e-4006e39fcf99: !Template - answer_choices: A ||| B - id: 6f84156e-22ae-4c07-8e3e-4006e39fcf99 + 5a1dfa62-42cb-4be9-a0ae-a29ba2242514: !Template + answer_choices: 1 ||| 2 + id: 5a1dfa62-42cb-4be9-a0ae-a29ba2242514 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -185,38 +170,53 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: B-A)' + name: grammatical_between_1_2 reference: '' - 721da667-24b0-4f42-be97-3974b69a0ff1: !Template - answer_choices: null - id: 721da667-24b0-4f42-be97-3974b69a0ff1 + 5f904965-a73a-4128-8749-32e36fd8ec1c: !Template + answer_choices: Yes ||| No + id: 5f904965-a73a-4128-8749-32e36fd8ec1c + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 634e3e9a-d706-4555-891f-38389954cec8: !Template + answer_choices: A ||| B + id: 634e3e9a-d706-4555-891f-38389954cec8 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -224,11 +224,27 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: better_A_B + reference: '' + dbd72c8a-e8c0-4804-a1dc-9d29aa54c325: !Template + answer_choices: Yes ||| No + id: dbd72c8a-e8c0-4804-a1dc-9d29aa54c325 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no reference: '' - bf0fe469-8193-412e-a789-64028bde8aaf: !Template + f67c53ae-ef19-4ccf-b608-68dd85fc279c: !Template answer_choices: Yes ||| No - id: bf0fe469-8193-412e-a789-64028bde8aaf + id: f67c53ae-ef19-4ccf-b608-68dd85fc279c jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -240,14 +256,14 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' + name: single_sentence_good_no_yes reference: '' - cfe57c2a-1359-485d-870d-7eafa36e86cb: !Template - answer_choices: null - id: cfe57c2a-1359-485d-870d-7eafa36e86cb + fa8140a0-a58c-48f1-883a-ab755fa17b37: !Template + answer_choices: A ||| B + id: fa8140a0-a58c-48f1-883a-ab755fa17b37 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -267,11 +283,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -279,21 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 - reference: '' - e6110306-ae02-43ff-b136-42b1dcc3a3d1: !Template - answer_choices: Yes ||| No - id: e6110306-ae02-43ff-b136-42b1dcc3a3d1 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' + name: grammatical_which_one_A_B reference: '' diff --git a/promptsource/templates/blimp/sentential_negation_npi_scope/templates.yaml b/promptsource/templates/blimp/sentential_negation_npi_scope/templates.yaml index 6607c07a2..147f0ddcb 100644 --- a/promptsource/templates/blimp/sentential_negation_npi_scope/templates.yaml +++ b/promptsource/templates/blimp/sentential_negation_npi_scope/templates.yaml @@ -1,64 +1,25 @@ dataset: blimp subset: sentential_negation_npi_scope templates: - 0724eef2-aff1-4c53-bd9a-8c76713f4936: !Template - answer_choices: null - id: 0724eef2-aff1-4c53-bd9a-8c76713f4936 - jinja: '{% set shuffled_order = [0, 1] | random %} - - Between the two sentences, which one is grammatical? Please answer 1 or 2. - - {% if shuffled_order == 0 %} - - 1: {{ sentence_good }} - - 2: {{ sentence_bad }} - - {% else %} - - 1: {{ sentence_bad }} - - 2: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - 1 - - {% else %} - - 2 - - {% endif %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: random_choice_1_2_2 - reference: '' - 2dc5b5e4-60f4-4f6e-97ca-715d00f3326e: !Template + 0be0526d-3819-49e6-8411-ab62400de1e5: !Template answer_choices: Yes ||| No - id: 2dc5b5e4-60f4-4f6e-97ca-715d00f3326e + id: 0be0526d-3819-49e6-8411-ab62400de1e5 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' + name: single_sentence_good_no_yes reference: '' - 2df0afd1-8c9a-4b0a-895a-3b7b93b22205: !Template + 0c9cdd4a-8ee4-4d49-bae6-e878cc85b1fb: !Template answer_choices: A ||| B - id: 2df0afd1-8c9a-4b0a-895a-3b7b93b22205 + id: 0c9cdd4a-8ee4-4d49-bae6-e878cc85b1fb jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -92,14 +53,14 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: A-B)' + name: better_A_B reference: '' - 6817fe38-c385-4902-91e1-bb424e2a6ee4: !Template - answer_choices: null - id: 6817fe38-c385-4902-91e1-bb424e2a6ee4 + 127e191c-83da-43a1-b5b4-0a2ba26e8808: !Template + answer_choices: 1 ||| 2 + id: 127e191c-83da-43a1-b5b4-0a2ba26e8808 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -119,11 +80,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -131,15 +92,15 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: grammatical_between_1_2 reference: '' - 8c1cfffd-03e9-40d4-b86b-9ac8b51c0a0c: !Template + 232a94b1-2f09-40c2-a89a-a3d497153252: !Template answer_choices: Yes ||| No - id: 8c1cfffd-03e9-40d4-b86b-9ac8b51c0a0c + id: 232a94b1-2f09-40c2-a89a-a3d497153252 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata @@ -147,14 +108,14 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' + name: single_sentence_bad_no_yes reference: '' - 908db614-72a9-488b-a880-c6656ddee630: !Template - answer_choices: null - id: 908db614-72a9-488b-a880-c6656ddee630 + 2b237dd0-d5f7-4781-9ff4-49330b863849: !Template + answer_choices: A ||| B + id: 2b237dd0-d5f7-4781-9ff4-49330b863849 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -174,11 +135,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -186,27 +147,50 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: grammatical_which_one_A_B reference: '' - a6484d4c-158f-4069-8ced-af9a48f49e15: !Template - answer_choices: Yes ||| No - id: a6484d4c-158f-4069-8ced-af9a48f49e15 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + 2b6a009d-9e14-42bc-98ce-825218c7e89c: !Template + answer_choices: A ||| B + id: 2b6a009d-9e14-42bc-98ce-825218c7e89c + jinja: '{% set shuffled_order = [0, 1] | random %} - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' + original_task: true + name: grammatical_between_A_B reference: '' - b2b3d98c-78fb-41b2-ab3a-2228f3743df7: !Template + 5fb90498-24a5-45d3-a65b-11e45b51f69c: !Template answer_choices: A ||| B - id: b2b3d98c-78fb-41b2-ab3a-2228f3743df7 + id: 5fb90498-24a5-45d3-a65b-11e45b51f69c jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -240,42 +224,42 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: B-A)' + name: better_B_A reference: '' - c30bc093-6b91-4da4-97ba-5369cac2d159: !Template + 76134d63-66d5-4c72-8e0f-6fbed4c6e363: !Template answer_choices: Yes ||| No - id: c30bc093-6b91-4da4-97ba-5369cac2d159 + id: 76134d63-66d5-4c72-8e0f-6fbed4c6e363 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' + name: single_sentence_bad_yes_no reference: '' - e2f937fb-bfa6-4c76-ada2-fb7b0f12a28b: !Template - answer_choices: null - id: e2f937fb-bfa6-4c76-ada2-fb7b0f12a28b + abaf5b42-4106-4327-9431-a3000475c589: !Template + answer_choices: 1 ||| 2 + id: abaf5b42-4106-4327-9431-a3000475c589 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -283,11 +267,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -295,5 +279,21 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: grammatical_which_one_1_2 + reference: '' + bfe1e980-cb40-4cac-97f9-a7acc923c8ea: !Template + answer_choices: Yes ||| No + id: bfe1e980-cb40-4cac-97f9-a7acc923c8ea + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no reference: '' diff --git a/promptsource/templates/blimp/sentential_subject_island/templates.yaml b/promptsource/templates/blimp/sentential_subject_island/templates.yaml index df4e65016..32fbda974 100644 --- a/promptsource/templates/blimp/sentential_subject_island/templates.yaml +++ b/promptsource/templates/blimp/sentential_subject_island/templates.yaml @@ -1,24 +1,40 @@ dataset: blimp subset: sentential_subject_island templates: - 008f1f04-9c60-4100-9596-1ae92b5f369d: !Template - answer_choices: null - id: 008f1f04-9c60-4100-9596-1ae92b5f369d + 057724f1-d841-4441-a7e0-4147de41894a: !Template + answer_choices: Yes ||| No + id: 057724f1-d841-4441-a7e0-4147de41894a + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 37f96a2a-25ec-4a71-b435-510d112c60a2: !Template + answer_choices: 1 ||| 2 + id: 37f96a2a-25ec-4a71-b435-510d112c60a2 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -26,11 +42,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -38,26 +54,26 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: grammatical_between_1_2 reference: '' - 14b7cba4-dbee-4fd1-a365-417cf47567ba: !Template - answer_choices: null - id: 14b7cba4-dbee-4fd1-a365-417cf47567ba + 3ef04d7b-83b1-45f6-ba19-5c992e177b11: !Template + answer_choices: A ||| B + id: 3ef04d7b-83b1-45f6-ba19-5c992e177b11 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -65,11 +81,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -77,15 +93,15 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: grammatical_which_one_A_B reference: '' - 301ca326-ff37-4415-8da8-8eff36300b83: !Template + 45674b88-247c-4d4f-89c2-34ae33616ad0: !Template answer_choices: Yes ||| No - id: 301ca326-ff37-4415-8da8-8eff36300b83 + id: 45674b88-247c-4d4f-89c2-34ae33616ad0 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata @@ -93,26 +109,26 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' + name: single_sentence_bad_no_yes reference: '' - 3e1817a6-288f-46fd-9304-a52dc801fcce: !Template - answer_choices: null - id: 3e1817a6-288f-46fd-9304-a52dc801fcce + 73864b31-6e30-48b8-91a8-7a4fead2cebf: !Template + answer_choices: 1 ||| 2 + id: 73864b31-6e30-48b8-91a8-7a4fead2cebf jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -120,11 +136,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -132,54 +148,37 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B - reference: '' - 3f717291-e579-4ffa-805e-185856f954fb: !Template - answer_choices: Yes ||| No - id: 3f717291-e579-4ffa-805e-185856f954fb - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' + name: grammatical_which_one_1_2 reference: '' - 4cddcaf0-86dc-4dfe-b3ab-9edd07b6fd23: !Template - answer_choices: null - id: 4cddcaf0-86dc-4dfe-b3ab-9edd07b6fd23 + 785928f5-c9bb-449c-be3a-e2c7ddd00f3f: !Template + answer_choices: A ||| B + id: 785928f5-c9bb-449c-be3a-e2c7ddd00f3f jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -187,47 +186,48 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: better_A_B reference: '' - 5728be55-6b7a-46c6-a2e9-fe0d2ff462e1: !Template + 81cf5f0f-fbe8-4ede-98ec-d2b8de738d20: !Template answer_choices: Yes ||| No - id: 5728be55-6b7a-46c6-a2e9-fe0d2ff462e1 + id: 81cf5f0f-fbe8-4ede-98ec-d2b8de738d20 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' + name: single_sentence_good_no_yes reference: '' - 8516d3eb-dc0d-4eb3-8324-544a7ca7e69a: !Template + 9bed3ec0-4e1b-4f7a-a222-655af91c2cf2: !Template answer_choices: A ||| B - id: 8516d3eb-dc0d-4eb3-8324-544a7ca7e69a + id: 9bed3ec0-4e1b-4f7a-a222-655af91c2cf2 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -241,11 +241,27 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: B-A)' + name: grammatical_between_A_B + reference: '' + dc7f52e0-3880-430f-b3bc-fb85b9688de4: !Template + answer_choices: Yes ||| No + id: dc7f52e0-3880-430f-b3bc-fb85b9688de4 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no reference: '' - 9239fcdb-429a-4c1f-9863-c7f42d3124a0: !Template + fecd0f72-b5ce-424b-a155-389da926817a: !Template answer_choices: A ||| B - id: 9239fcdb-429a-4c1f-9863-c7f42d3124a0 + id: fecd0f72-b5ce-424b-a155-389da926817a jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -264,7 +280,7 @@ templates: {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order + Which one is a better sentence of English, B or A? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -279,21 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: A-B)' - reference: '' - cdb3bc9b-2645-415b-b5a5-6e607f6bae8d: !Template - answer_choices: Yes ||| No - id: cdb3bc9b-2645-415b-b5a5-6e607f6bae8d - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' + name: better_B_A reference: '' diff --git a/promptsource/templates/blimp/superlative_quantifiers_1/templates.yaml b/promptsource/templates/blimp/superlative_quantifiers_1/templates.yaml index 155bd5401..c72c7130b 100644 --- a/promptsource/templates/blimp/superlative_quantifiers_1/templates.yaml +++ b/promptsource/templates/blimp/superlative_quantifiers_1/templates.yaml @@ -1,36 +1,35 @@ dataset: blimp subset: superlative_quantifiers_1 templates: - 4f251422-3bf3-448f-90c6-b3d7ff0d8cea: !Template - answer_choices: null - id: 4f251422-3bf3-448f-90c6-b3d7ff0d8cea + 2a069f15-0c13-431e-8447-ecc471255aa8: !Template + answer_choices: A ||| B + id: 2a069f15-0c13-431e-8447-ecc471255aa8 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -38,58 +37,26 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: better_B_A reference: '' - 5fc50de0-8349-4007-b7d4-a9cae68ad67f: !Template - answer_choices: Yes ||| No - id: 5fc50de0-8349-4007-b7d4-a9cae68ad67f - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' - reference: '' - 69c7fc58-4bf3-4ab4-b691-3ee9138e3e3a: !Template - answer_choices: Yes ||| No - id: 69c7fc58-4bf3-4ab4-b691-3ee9138e3e3a - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' - reference: '' - 872c75f1-7513-4a56-8d99-351a5c62a80d: !Template - answer_choices: null - id: 872c75f1-7513-4a56-8d99-351a5c62a80d + 2a3bb462-021d-46b0-94ee-a382d54ea3bc: !Template + answer_choices: 1 ||| 2 + id: 2a3bb462-021d-46b0-94ee-a382d54ea3bc jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -97,11 +64,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -109,31 +76,48 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: grammatical_between_1_2 reference: '' - ae0069dc-d181-4e60-8c77-629d8b8c3c24: !Template + 2c870ca2-d51d-45c9-acee-7d92deefd54c: !Template + answer_choices: Yes ||| No + id: 2c870ca2-d51d-45c9-acee-7d92deefd54c + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes + reference: '' + 35f1d980-1a3d-4407-a731-d7ec801f594e: !Template answer_choices: A ||| B - id: ae0069dc-d181-4e60-8c77-629d8b8c3c24 + id: 35f1d980-1a3d-4407-a731-d7ec801f594e jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -147,15 +131,15 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: A-B)' + name: grammatical_which_one_A_B reference: '' - b563c6ed-392b-402f-a8ae-ea2ca019446e: !Template + 3eeabe94-4b00-465a-810a-e05e63a62a84: !Template answer_choices: Yes ||| No - id: b563c6ed-392b-402f-a8ae-ea2ca019446e + id: 3eeabe94-4b00-465a-810a-e05e63a62a84 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata @@ -163,11 +147,11 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' + name: single_sentence_bad_yes_no reference: '' - dab9ed24-e1dd-452e-8999-5231909b5f36: !Template - answer_choices: null - id: dab9ed24-e1dd-452e-8999-5231909b5f36 + 4e5615fa-52dc-49fe-a0e2-56236b4e733d: !Template + answer_choices: 1 ||| 2 + id: 4e5615fa-52dc-49fe-a0e2-56236b4e733d jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer 1 or 2. @@ -190,11 +174,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -202,11 +186,27 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: grammatical_which_one_1_2 + reference: '' + 84abb131-62a8-4f62-b76e-b4d53807fa75: !Template + answer_choices: Yes ||| No + id: 84abb131-62a8-4f62-b76e-b4d53807fa75 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes reference: '' - f474ffc2-075c-4ff0-af37-491a8034cec5: !Template + 917ef9e0-02ab-40c4-a126-5177a02523ea: !Template answer_choices: A ||| B - id: f474ffc2-075c-4ff0-af37-491a8034cec5 + id: 917ef9e0-02ab-40c4-a126-5177a02523ea jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -225,7 +225,7 @@ templates: {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order + Which one is a better sentence of English, A or B? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -240,42 +240,26 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: B-A)' + name: better_A_B reference: '' - f8847769-33b1-42cd-ae19-b1a537bfa066: !Template - answer_choices: Yes ||| No - id: f8847769-33b1-42cd-ae19-b1a537bfa066 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' - reference: '' - fd169ab7-3b97-415c-a485-a5a71f915bf4: !Template - answer_choices: null - id: fd169ab7-3b97-415c-a485-a5a71f915bf4 + 9cc03745-3cdf-4e84-bf41-47dd4d114ee0: !Template + answer_choices: A ||| B + id: 9cc03745-3cdf-4e84-bf41-47dd4d114ee0 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -283,11 +267,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -295,5 +279,21 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: grammatical_between_A_B + reference: '' + 9ce8c6ac-061c-47ce-aff3-43ba68d41081: !Template + answer_choices: Yes ||| No + id: 9ce8c6ac-061c-47ce-aff3-43ba68d41081 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no reference: '' diff --git a/promptsource/templates/blimp/superlative_quantifiers_2/templates.yaml b/promptsource/templates/blimp/superlative_quantifiers_2/templates.yaml index f886173d3..c708f7074 100644 --- a/promptsource/templates/blimp/superlative_quantifiers_2/templates.yaml +++ b/promptsource/templates/blimp/superlative_quantifiers_2/templates.yaml @@ -1,29 +1,62 @@ dataset: blimp subset: superlative_quantifiers_2 templates: - 1149a7af-e509-411b-80be-192905e047d4: !Template + 2aae800b-a6cd-46fc-a518-fbc5ef126337: !Template + answer_choices: Yes ||| No + id: 2aae800b-a6cd-46fc-a518-fbc5ef126337 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 2f4e0d4c-0a1e-4a70-aadb-a8793def2ed6: !Template + answer_choices: Yes ||| No + id: 2f4e0d4c-0a1e-4a70-aadb-a8793def2ed6 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes + reference: '' + 342f2622-44f7-4df2-900c-1bff71b37dad: !Template answer_choices: A ||| B - id: 1149a7af-e509-411b-80be-192905e047d4 + id: 342f2622-44f7-4df2-900c-1bff71b37dad jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -37,38 +70,37 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: A-B)' + name: grammatical_between_A_B reference: '' - 263d68e2-e44e-4f8f-8b1d-487b2aa70b3a: !Template - answer_choices: null - id: 263d68e2-e44e-4f8f-8b1d-487b2aa70b3a + 3ab2784f-f409-4041-9472-c454bc26b0cb: !Template + answer_choices: A ||| B + id: 3ab2784f-f409-4041-9472-c454bc26b0cb jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -76,42 +108,42 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: better_B_A reference: '' - 32119cbc-b09a-4b0d-b9c4-687b55e7c696: !Template + 7a7242c8-f0aa-4a44-8b0e-9a0c098e3504: !Template answer_choices: Yes ||| No - id: 32119cbc-b09a-4b0d-b9c4-687b55e7c696 + id: 7a7242c8-f0aa-4a44-8b0e-9a0c098e3504 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' + name: single_sentence_good_no_yes reference: '' - 522e13a1-fcd9-4540-bba0-a4439715cc1d: !Template - answer_choices: null - id: 522e13a1-fcd9-4540-bba0-a4439715cc1d + 7c7ceb51-7cbf-48d2-9fe7-0e5f56d2767b: !Template + answer_choices: A ||| B + id: 7c7ceb51-7cbf-48d2-9fe7-0e5f56d2767b jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -119,11 +151,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -131,31 +163,32 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: grammatical_which_one_A_B reference: '' - 59110d8e-f7f9-4d26-b72d-3fbfc9ed9970: !Template - answer_choices: A ||| B - id: 59110d8e-f7f9-4d26-b72d-3fbfc9ed9970 + 7e4ec882-cd79-478d-86f5-68ca8f726343: !Template + answer_choices: 1 ||| 2 + id: 7e4ec882-cd79-478d-86f5-68ca8f726343 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -169,26 +202,26 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: B-A)' + name: grammatical_which_one_1_2 reference: '' - 903bf9f1-4b0b-4f05-acb6-fb2bcdb876c0: !Template - answer_choices: null - id: 903bf9f1-4b0b-4f05-acb6-fb2bcdb876c0 + a03a7186-d11b-4a9a-8a6f-edb965803b0e: !Template + answer_choices: 1 ||| 2 + id: a03a7186-d11b-4a9a-8a6f-edb965803b0e jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -196,11 +229,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -208,15 +241,15 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: grammatical_between_1_2 reference: '' - ad315691-8475-426d-b61c-754b4cfe0b4d: !Template + b13b6751-8ef0-427a-8a85-67302fdcaaac: !Template answer_choices: Yes ||| No - id: ad315691-8475-426d-b61c-754b4cfe0b4d + id: b13b6751-8ef0-427a-8a85-67302fdcaaac jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata @@ -224,38 +257,37 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' + name: single_sentence_bad_yes_no reference: '' - c0d98b6b-c484-4537-81be-292d6ef0496b: !Template - answer_choices: null - id: c0d98b6b-c484-4537-81be-292d6ef0496b + e358818b-d4e2-4b13-b471-1b3efa8db9e7: !Template + answer_choices: A ||| B + id: e358818b-d4e2-4b13-b471-1b3efa8db9e7 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -263,37 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 - reference: '' - d74fba72-6ac3-455f-9459-ebf52c93e5a8: !Template - answer_choices: Yes ||| No - id: d74fba72-6ac3-455f-9459-ebf52c93e5a8 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' - reference: '' - f9d185b6-78ed-4b5c-97cc-f73c2242e3a2: !Template - answer_choices: Yes ||| No - id: f9d185b6-78ed-4b5c-97cc-f73c2242e3a2 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' + name: better_A_B reference: '' diff --git a/promptsource/templates/blimp/tough_vs_raising_1/templates.yaml b/promptsource/templates/blimp/tough_vs_raising_1/templates.yaml index 4439add8e..1487c95c4 100644 --- a/promptsource/templates/blimp/tough_vs_raising_1/templates.yaml +++ b/promptsource/templates/blimp/tough_vs_raising_1/templates.yaml @@ -1,36 +1,35 @@ dataset: blimp subset: tough_vs_raising_1 templates: - 0528d81f-ab0b-47c8-a7bd-80dc4b6d84d8: !Template - answer_choices: null - id: 0528d81f-ab0b-47c8-a7bd-80dc4b6d84d8 + 11f1d6e4-fecb-43e6-82e1-d1bcb9d42b60: !Template + answer_choices: A ||| B + id: 11f1d6e4-fecb-43e6-82e1-d1bcb9d42b60 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -38,11 +37,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: better_B_A reference: '' - 0745a26d-37a7-4ef1-b544-26994e3eb1d2: !Template - answer_choices: null - id: 0745a26d-37a7-4ef1-b544-26994e3eb1d2 + 4f4787f9-a285-4438-b7ad-a83aabfde1d3: !Template + answer_choices: A ||| B + id: 4f4787f9-a285-4438-b7ad-a83aabfde1d3 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer A or B. @@ -65,11 +64,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -77,11 +76,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: grammatical_between_A_B reference: '' - 183a789a-ec55-40e1-b0af-2f30532ec07b: !Template - answer_choices: null - id: 183a789a-ec55-40e1-b0af-2f30532ec07b + 6f925c37-203b-431c-86e5-f7e96ffaaae6: !Template + answer_choices: 1 ||| 2 + id: 6f925c37-203b-431c-86e5-f7e96ffaaae6 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer 1 or 2. @@ -104,11 +103,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -116,31 +115,48 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: grammatical_which_one_1_2 reference: '' - 1e641e47-4f12-4351-9f93-467e6ce44a02: !Template - answer_choices: A ||| B - id: 1e641e47-4f12-4351-9f93-467e6ce44a02 + 71893fff-522d-4cb3-8fef-6d34f6fd969b: !Template + answer_choices: Yes ||| No + id: 71893fff-522d-4cb3-8fef-6d34f6fd969b + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 7debf4a9-23ca-45fa-a67d-2103ae2336a9: !Template + answer_choices: 1 ||| 2 + id: 7debf4a9-23ca-45fa-a67d-2103ae2336a9 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -154,11 +170,11 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: A-B)' + name: grammatical_between_1_2 reference: '' - 7b5b1b06-7029-4e51-a501-385d8e5a6f8a: !Template - answer_choices: null - id: 7b5b1b06-7029-4e51-a501-385d8e5a6f8a + 7e869d1a-29e8-43cf-b4e9-1e128d5acd95: !Template + answer_choices: A ||| B + id: 7e869d1a-29e8-43cf-b4e9-1e128d5acd95 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer A or B. @@ -181,11 +197,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -193,43 +209,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: grammatical_which_one_A_B reference: '' - 8cfb32b8-4459-4b89-8285-780d457b9f72: !Template + b5c49924-ca7f-43b2-86b1-588a5d6915e2: !Template answer_choices: Yes ||| No - id: 8cfb32b8-4459-4b89-8285-780d457b9f72 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' - reference: '' - 9306abb3-5707-4fb8-8ad7-4fbd5329906d: !Template - answer_choices: Yes ||| No - id: 9306abb3-5707-4fb8-8ad7-4fbd5329906d - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' - reference: '' - 9b13b45f-1b3c-4a40-ae7f-b1ab8b57a781: !Template - answer_choices: Yes ||| No - id: 9b13b45f-1b3c-4a40-ae7f-b1ab8b57a781 + id: b5c49924-ca7f-43b2-86b1-588a5d6915e2 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -241,11 +225,11 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' + name: single_sentence_good_no_yes reference: '' - af7dd32a-7696-447d-addb-3fad35c9284f: !Template + cb656ca1-9b29-4598-9231-7ba2c793565b: !Template answer_choices: A ||| B - id: af7dd32a-7696-447d-addb-3fad35c9284f + id: cb656ca1-9b29-4598-9231-7ba2c793565b jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -264,7 +248,7 @@ templates: {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order + Which one is a better sentence of English, A or B? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -279,15 +263,15 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: B-A)' + name: better_A_B reference: '' - e64a405e-504b-4de6-98c7-567c04986b6b: !Template + cfd001eb-0f98-4f91-afac-6151e6ea3294: !Template answer_choices: Yes ||| No - id: e64a405e-504b-4de6-98c7-567c04986b6b + id: cfd001eb-0f98-4f91-afac-6151e6ea3294 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata @@ -295,5 +279,21 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' + name: single_sentence_bad_no_yes + reference: '' + e1b6ebf4-b1eb-4a96-a644-a5c918cf85f8: !Template + answer_choices: Yes ||| No + id: e1b6ebf4-b1eb-4a96-a644-a5c918cf85f8 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no reference: '' diff --git a/promptsource/templates/blimp/tough_vs_raising_2/templates.yaml b/promptsource/templates/blimp/tough_vs_raising_2/templates.yaml index dd88093dd..0ea200f30 100644 --- a/promptsource/templates/blimp/tough_vs_raising_2/templates.yaml +++ b/promptsource/templates/blimp/tough_vs_raising_2/templates.yaml @@ -1,9 +1,9 @@ dataset: blimp subset: tough_vs_raising_2 templates: - 0d54febd-41ae-4fbd-8ca7-ef5aac51453c: !Template + 05b504b6-1c19-4083-ad26-cecc6afd1363: !Template answer_choices: A ||| B - id: 0d54febd-41ae-4fbd-8ca7-ef5aac51453c + id: 05b504b6-1c19-4083-ad26-cecc6afd1363 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -37,11 +37,11 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: B-A)' + name: better_B_A reference: '' - 22580638-cf0c-46c2-97dd-7ffea779d0a6: !Template - answer_choices: null - id: 22580638-cf0c-46c2-97dd-7ffea779d0a6 + 13dedd40-8c4a-45b7-949c-2acd071b1821: !Template + answer_choices: 1 ||| 2 + id: 13dedd40-8c4a-45b7-949c-2acd071b1821 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -64,11 +64,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -76,42 +76,42 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: grammatical_between_1_2 reference: '' - 581810cc-0477-4179-a8b0-65954d68e802: !Template + 1ca42122-f025-4a5d-8f58-d2c9771a55fa: !Template answer_choices: Yes ||| No - id: 581810cc-0477-4179-a8b0-65954d68e802 + id: 1ca42122-f025-4a5d-8f58-d2c9771a55fa jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' + name: single_sentence_good_no_yes reference: '' - 5a1dfbf7-feb3-4457-be61-3c095b91e426: !Template - answer_choices: null - id: 5a1dfbf7-feb3-4457-be61-3c095b91e426 + 273ca188-a5d2-4cb5-a9d2-6c5adc8ae087: !Template + answer_choices: A ||| B + id: 273ca188-a5d2-4cb5-a9d2-6c5adc8ae087 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -119,11 +119,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -131,50 +131,27 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: grammatical_which_one_A_B reference: '' - 5a9680a8-eaaf-48ef-909a-0260461629fa: !Template - answer_choices: null - id: 5a9680a8-eaaf-48ef-909a-0260461629fa - jinja: '{% set shuffled_order = [0, 1] | random %} - - Which one of the following sentences is grammatical? Please answer A or B. - - {% if shuffled_order == 0 %} - - A: {{ sentence_good }} - - B: {{ sentence_bad }} - - {% else %} - - A: {{ sentence_bad }} - - B: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - A - - {% else %} - - B + 357d62f1-fb86-48c3-a1b0-69c71b5e5049: !Template + answer_choices: Yes ||| No + id: 357d62f1-fb86-48c3-a1b0-69c71b5e5049 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - {% endif %}' + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: random_choice_A_B + original_task: false + name: single_sentence_good_yes_no reference: '' - 7ae170fa-ddac-4d62-bd23-5af95cbd8b59: !Template + 97f22748-e724-4f87-a17f-2a7a6aae80d0: !Template answer_choices: A ||| B - id: 7ae170fa-ddac-4d62-bd23-5af95cbd8b59 + id: 97f22748-e724-4f87-a17f-2a7a6aae80d0 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -208,11 +185,27 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: A-B)' + name: better_A_B + reference: '' + ac5a8915-c5bb-449e-aa5f-4cb9a422876a: !Template + answer_choices: Yes ||| No + id: ac5a8915-c5bb-449e-aa5f-4cb9a422876a + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no reference: '' - 9845e5d4-cb4e-4149-aabe-e4a73ad5b93a: !Template - answer_choices: null - id: 9845e5d4-cb4e-4149-aabe-e4a73ad5b93a + c83e59e0-b51a-4bcb-9bfb-baae15b0b9d2: !Template + answer_choices: A ||| B + id: c83e59e0-b51a-4bcb-9bfb-baae15b0b9d2 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer A or B. @@ -235,11 +228,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -247,15 +240,15 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: grammatical_between_A_B reference: '' - b782d11e-a941-4ade-85af-93756a53297b: !Template + ede13f24-7d23-49b8-8d00-83cb4b2a99e2: !Template answer_choices: Yes ||| No - id: b782d11e-a941-4ade-85af-93756a53297b + id: ede13f24-7d23-49b8-8d00-83cb4b2a99e2 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata @@ -263,37 +256,44 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' + name: single_sentence_bad_no_yes reference: '' - c1814309-3ce7-400e-8aef-94f465f2f0d8: !Template - answer_choices: Yes ||| No - id: c1814309-3ce7-400e-8aef-94f465f2f0d8 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + f5bc4220-7cac-451a-9886-440911243a54: !Template + answer_choices: 1 ||| 2 + id: f5bc4220-7cac-451a-9886-440911243a54 + jinja: '{% set shuffled_order = [0, 1] | random %} - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' - reference: '' - de22063f-3ae4-442d-877f-d9326e90de27: !Template - answer_choices: Yes ||| No - id: de22063f-3ae4-442d-877f-d9326e90de27 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + Which one of the following sentences is grammatical? Please answer 1 or 2. - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' + original_task: true + name: grammatical_which_one_1_2 reference: '' diff --git a/promptsource/templates/blimp/transitive/templates.yaml b/promptsource/templates/blimp/transitive/templates.yaml index 39a1ef564..e3e73b782 100644 --- a/promptsource/templates/blimp/transitive/templates.yaml +++ b/promptsource/templates/blimp/transitive/templates.yaml @@ -1,9 +1,41 @@ dataset: blimp subset: transitive templates: - 0481dc81-b5d6-4003-9f23-8b64a49d5296: !Template + 2304b1e7-d649-4d97-8806-d29282922ce6: !Template + answer_choices: Yes ||| No + id: 2304b1e7-d649-4d97-8806-d29282922ce6 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 43d3db4f-08e1-487f-a8ae-977c38b90a5e: !Template + answer_choices: Yes ||| No + id: 43d3db4f-08e1-487f-a8ae-977c38b90a5e + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 4f6d3983-521b-4b51-812e-1ce2532e4ede: !Template answer_choices: A ||| B - id: 0481dc81-b5d6-4003-9f23-8b64a49d5296 + id: 4f6d3983-521b-4b51-812e-1ce2532e4ede jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -22,7 +54,7 @@ templates: {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order + Which one is a better sentence of English, B or A? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -37,31 +69,32 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: A-B)' + name: better_B_A reference: '' - 08a59f81-6b04-4f9a-bcfe-4a5dd9b2ba70: !Template + 65ee533c-1889-421e-810b-7bfd6a80e49a: !Template answer_choices: A ||| B - id: 08a59f81-6b04-4f9a-bcfe-4a5dd9b2ba70 + id: 65ee533c-1889-421e-810b-7bfd6a80e49a jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -75,11 +108,11 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: B-A)' + name: grammatical_which_one_A_B reference: '' - 20291886-1104-435d-af59-62f92208ab8a: !Template + 828a65c0-fffc-4daa-98e8-6bee5ddbe725: !Template answer_choices: Yes ||| No - id: 20291886-1104-435d-af59-62f92208ab8a + id: 828a65c0-fffc-4daa-98e8-6bee5ddbe725 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -91,58 +124,26 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' + name: single_sentence_good_no_yes reference: '' - 218a0707-57b1-42f4-9467-da6c3f839d12: !Template - answer_choices: Yes ||| No - id: 218a0707-57b1-42f4-9467-da6c3f839d12 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' - reference: '' - 3b12589b-4746-442b-a2e3-001f9e8ac024: !Template - answer_choices: Yes ||| No - id: 3b12589b-4746-442b-a2e3-001f9e8ac024 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' - reference: '' - 3b247a68-a7ea-4330-ac9c-0b3c330fd2a2: !Template - answer_choices: null - id: 3b247a68-a7ea-4330-ac9c-0b3c330fd2a2 + 9c5a2a72-bf6d-463c-a463-576badc80558: !Template + answer_choices: 1 ||| 2 + id: 9c5a2a72-bf6d-463c-a463-576badc80558 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -150,11 +151,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -162,11 +163,27 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: grammatical_between_1_2 reference: '' - 4042fd6c-ea58-4f35-98ce-d6511fbf6428: !Template - answer_choices: null - id: 4042fd6c-ea58-4f35-98ce-d6511fbf6428 + a8b45ec5-64b2-465e-a371-775fe0e8e8e7: !Template + answer_choices: Yes ||| No + id: a8b45ec5-64b2-465e-a371-775fe0e8e8e7 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes + reference: '' + aa2d5917-5c3c-453e-ac3f-936ba57a5a2d: !Template + answer_choices: 1 ||| 2 + id: aa2d5917-5c3c-453e-ac3f-936ba57a5a2d jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer 1 or 2. @@ -189,11 +206,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -201,38 +218,37 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: grammatical_which_one_1_2 reference: '' - 4c3ea778-4682-43f5-9169-986a1127e8a2: !Template - answer_choices: null - id: 4c3ea778-4682-43f5-9169-986a1127e8a2 + ae9210af-cfd5-47e0-b321-15397f59ffcd: !Template + answer_choices: A ||| B + id: ae9210af-cfd5-47e0-b321-15397f59ffcd jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -240,26 +256,26 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: better_A_B reference: '' - b8342f4e-ddff-44d4-a153-4515e4bf381f: !Template - answer_choices: null - id: b8342f4e-ddff-44d4-a153-4515e4bf381f + d9af92c2-9c54-4d14-a4c0-aa8e48e18dd3: !Template + answer_choices: A ||| B + id: d9af92c2-9c54-4d14-a4c0-aa8e48e18dd3 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -267,11 +283,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -279,21 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 - reference: '' - e6ab77e5-bca0-430a-93d4-6973b63a221a: !Template - answer_choices: Yes ||| No - id: e6ab77e5-bca0-430a-93d4-6973b63a221a - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' + name: grammatical_between_A_B reference: '' diff --git a/promptsource/templates/blimp/wh_island/templates.yaml b/promptsource/templates/blimp/wh_island/templates.yaml index e76af84e6..59022755e 100644 --- a/promptsource/templates/blimp/wh_island/templates.yaml +++ b/promptsource/templates/blimp/wh_island/templates.yaml @@ -1,52 +1,35 @@ dataset: blimp subset: wh_island templates: - 04f03f62-9cf6-46cd-a4e9-e7b48e240825: !Template - answer_choices: Yes ||| No - id: 04f03f62-9cf6-46cd-a4e9-e7b48e240825 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' - reference: '' - 0ba65f3a-fe83-4b11-9840-e8d7f25c0fad: !Template - answer_choices: null - id: 0ba65f3a-fe83-4b11-9840-e8d7f25c0fad + 1277e2b2-f41d-4f95-a752-52fd92213676: !Template + answer_choices: A ||| B + id: 1277e2b2-f41d-4f95-a752-52fd92213676 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -54,11 +37,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: better_A_B reference: '' - 0dba9bb2-8fad-4aa0-9c4f-269050428da8: !Template + 196d9fd5-f091-4ea2-afa7-29a6523df0d2: !Template answer_choices: A ||| B - id: 0dba9bb2-8fad-4aa0-9c4f-269050428da8 + id: 196d9fd5-f091-4ea2-afa7-29a6523df0d2 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -77,7 +60,7 @@ templates: {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order + Which one is a better sentence of English, B or A? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -92,26 +75,26 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: A-B)' + name: better_B_A reference: '' - 32562d86-9a21-4c06-9ae5-9e080e53bd04: !Template - answer_choices: null - id: 32562d86-9a21-4c06-9ae5-9e080e53bd04 + 31d1b252-0b79-4742-aa0a-2303a5b5b43a: !Template + answer_choices: A ||| B + id: 31d1b252-0b79-4742-aa0a-2303a5b5b43a jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -119,11 +102,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -131,31 +114,32 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: grammatical_which_one_A_B reference: '' - 3cf67ba6-3dc6-4c76-afb4-f0b7dd5a1d78: !Template + 8e4060b6-7786-4714-9cc4-50bb61a5f791: !Template answer_choices: A ||| B - id: 3cf67ba6-3dc6-4c76-afb4-f0b7dd5a1d78 + id: 8e4060b6-7786-4714-9cc4-50bb61a5f791 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -169,11 +153,27 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: B-A)' + name: grammatical_between_A_B reference: '' - 5447fec4-045c-4578-a120-749f4c308462: !Template - answer_choices: null - id: 5447fec4-045c-4578-a120-749f4c308462 + 941969d9-b0e5-4eea-9295-b2286c18b3ed: !Template + answer_choices: Yes ||| No + id: 941969d9-b0e5-4eea-9295-b2286c18b3ed + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes + reference: '' + a4ac23ce-b4c3-4202-a519-6931868eb504: !Template + answer_choices: 1 ||| 2 + id: a4ac23ce-b4c3-4202-a519-6931868eb504 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer 1 or 2. @@ -196,11 +196,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -208,42 +208,26 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: grammatical_which_one_1_2 reference: '' - c43490fa-7c7e-415b-ad5d-f449b571b261: !Template - answer_choices: Yes ||| No - id: c43490fa-7c7e-415b-ad5d-f449b571b261 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' - reference: '' - c56d413e-ac51-412c-a2d3-11d66a3559b3: !Template - answer_choices: null - id: c56d413e-ac51-412c-a2d3-11d66a3559b3 + bbc60b2c-f5e3-4c95-9b5a-04f67e373064: !Template + answer_choices: 1 ||| 2 + id: bbc60b2c-f5e3-4c95-9b5a-04f67e373064 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -251,11 +235,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -263,11 +247,27 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: grammatical_between_1_2 + reference: '' + befe5d14-82e7-4795-ad81-96109373982f: !Template + answer_choices: Yes ||| No + id: befe5d14-82e7-4795-ad81-96109373982f + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no reference: '' - d7146feb-1a26-44ae-af5a-00595cf9f210: !Template + c8f0b100-c673-4703-afb2-0aa47511360c: !Template answer_choices: Yes ||| No - id: d7146feb-1a26-44ae-af5a-00595cf9f210 + id: c8f0b100-c673-4703-afb2-0aa47511360c jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -279,11 +279,11 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' + name: single_sentence_bad_no_yes reference: '' - e7be32ff-90ce-4472-9917-4075baa411a7: !Template + e4a84272-1b18-4db0-8d22-eafb4032f5e5: !Template answer_choices: Yes ||| No - id: e7be32ff-90ce-4472-9917-4075baa411a7 + id: e4a84272-1b18-4db0-8d22-eafb4032f5e5 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -295,5 +295,5 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' + name: single_sentence_good_yes_no reference: '' diff --git a/promptsource/templates/blimp/wh_questions_object_gap/templates.yaml b/promptsource/templates/blimp/wh_questions_object_gap/templates.yaml index d23e4646f..071ed4d45 100644 --- a/promptsource/templates/blimp/wh_questions_object_gap/templates.yaml +++ b/promptsource/templates/blimp/wh_questions_object_gap/templates.yaml @@ -1,25 +1,48 @@ dataset: blimp subset: wh_questions_object_gap templates: - 057ce505-36bc-4c6a-873f-7ead936f3a33: !Template - answer_choices: Yes ||| No - id: 057ce505-36bc-4c6a-873f-7ead936f3a33 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + 0ba81e3c-e360-48b7-98fd-2ef7955c1607: !Template + answer_choices: A ||| B + id: 0ba81e3c-e360-48b7-98fd-2ef7955c1607 + jinja: '{% set shuffled_order = [0, 1] | random %} - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' + original_task: true + name: grammatical_which_one_A_B reference: '' - 0cd48e77-78d8-4c47-b051-9890e583eda6: !Template + 16dcde34-9933-464a-8927-7556d034db0f: !Template answer_choices: Yes ||| No - id: 0cd48e77-78d8-4c47-b051-9890e583eda6 + id: 16dcde34-9933-464a-8927-7556d034db0f jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -31,11 +54,11 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' + name: single_sentence_good_no_yes reference: '' - 3512ec57-5b8c-4a06-b375-173096c697d6: !Template - answer_choices: null - id: 3512ec57-5b8c-4a06-b375-173096c697d6 + 3bbe15f8-6ded-4aa1-beea-6adfb9bbf323: !Template + answer_choices: A ||| B + id: 3bbe15f8-6ded-4aa1-beea-6adfb9bbf323 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer A or B. @@ -58,11 +81,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -70,11 +93,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: grammatical_between_A_B reference: '' - 5c17a440-b0ff-4d27-b6d1-535b937f6348: !Template - answer_choices: null - id: 5c17a440-b0ff-4d27-b6d1-535b937f6348 + 670e3dae-14cd-46a9-a356-77ad86e317b1: !Template + answer_choices: 1 ||| 2 + id: 670e3dae-14cd-46a9-a356-77ad86e317b1 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -97,11 +120,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -109,27 +132,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: grammatical_between_1_2 reference: '' - 5e9c971d-520b-4e0b-8442-e0bb719f40ec: !Template - answer_choices: Yes ||| No - id: 5e9c971d-520b-4e0b-8442-e0bb719f40ec - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' - reference: '' - b29436b6-37e5-4915-a91d-991c61c30951: !Template + 76324e7f-cda6-4e89-ad8e-817fbe507b44: !Template answer_choices: A ||| B - id: b29436b6-37e5-4915-a91d-991c61c30951 + id: 76324e7f-cda6-4e89-ad8e-817fbe507b44 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -163,50 +170,59 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: B-A)' + name: better_B_A reference: '' - c2dbc938-a331-4127-b66c-3d3be7074239: !Template - answer_choices: null - id: c2dbc938-a331-4127-b66c-3d3be7074239 - jinja: '{% set shuffled_order = [0, 1] | random %} - - Which one of the following sentences is grammatical? Please answer A or B. - - {% if shuffled_order == 0 %} - - A: {{ sentence_good }} - - B: {{ sentence_bad }} - - {% else %} - - A: {{ sentence_bad }} - - B: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - A + 87415348-9aa3-4cdb-9600-0e48d5e18ee8: !Template + answer_choices: Yes ||| No + id: 87415348-9aa3-4cdb-9600-0e48d5e18ee8 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - {% else %} + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes + reference: '' + 91f5de4d-2392-4230-9cfb-41237333f246: !Template + answer_choices: Yes ||| No + id: 91f5de4d-2392-4230-9cfb-41237333f246 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - B + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + ad9ce087-f787-46aa-936e-08f09a2b3b6b: !Template + answer_choices: Yes ||| No + id: ad9ce087-f787-46aa-936e-08f09a2b3b6b + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - {% endif %}' + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: random_choice_A_B + original_task: false + name: single_sentence_good_yes_no reference: '' - de5a8942-bc5a-4138-912f-142eef1f11f2: !Template + b8da5dfa-5eaa-4140-a864-a288371a71ff: !Template answer_choices: A ||| B - id: de5a8942-bc5a-4138-912f-142eef1f11f2 + id: b8da5dfa-5eaa-4140-a864-a288371a71ff jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -240,27 +256,11 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: A-B)' + name: better_A_B reference: '' - e652d0f3-a32f-4f31-ba83-afd6408fa40f: !Template - answer_choices: Yes ||| No - id: e652d0f3-a32f-4f31-ba83-afd6408fa40f - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' - reference: '' - fc3c593f-7bea-43e1-96ad-ef1fa8a7fb16: !Template - answer_choices: null - id: fc3c593f-7bea-43e1-96ad-ef1fa8a7fb16 + ce574003-2e89-4e0c-a36b-85c8ae02560a: !Template + answer_choices: 1 ||| 2 + id: ce574003-2e89-4e0c-a36b-85c8ae02560a jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer 1 or 2. @@ -283,11 +283,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -295,5 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: grammatical_which_one_1_2 reference: '' diff --git a/promptsource/templates/blimp/wh_questions_subject_gap/templates.yaml b/promptsource/templates/blimp/wh_questions_subject_gap/templates.yaml index 0dfdbf594..119de5aa3 100644 --- a/promptsource/templates/blimp/wh_questions_subject_gap/templates.yaml +++ b/promptsource/templates/blimp/wh_questions_subject_gap/templates.yaml @@ -1,9 +1,25 @@ dataset: blimp subset: wh_questions_subject_gap templates: - 0dfdb9b5-374e-443f-82c2-3498e7ef10ed: !Template + 3be2fbd3-dc8a-42a8-b75f-2bf5f97a9a06: !Template + answer_choices: Yes ||| No + id: 3be2fbd3-dc8a-42a8-b75f-2bf5f97a9a06 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes + reference: '' + 3e74107b-4b89-4a8e-8e87-5b58ea940035: !Template answer_choices: A ||| B - id: 0dfdb9b5-374e-443f-82c2-3498e7ef10ed + id: 3e74107b-4b89-4a8e-8e87-5b58ea940035 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -22,7 +38,7 @@ templates: {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order + Which one is a better sentence of English, A or B? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -37,70 +53,76 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: B-A)' + name: better_A_B reference: '' - 1df0cb41-8c9e-4e84-8146-c6f8a586e7fc: !Template - answer_choices: Yes ||| No - id: 1df0cb41-8c9e-4e84-8146-c6f8a586e7fc - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + 517be3cd-115a-4310-bf56-53c41e1544da: !Template + answer_choices: A ||| B + id: 517be3cd-115a-4310-bf56-53c41e1544da + jinja: '{% set shuffled_order = [0, 1] | random %} - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' - reference: '' - 36e5bebe-4cf7-47db-890d-c53fff0e1e09: !Template - answer_choices: Yes ||| No - id: 36e5bebe-4cf7-47db-890d-c53fff0e1e09 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + Which one of the following sentences is grammatical? Please answer A or B. - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' + original_task: true + name: grammatical_which_one_A_B reference: '' - 53bb4f17-cc2c-40ee-b696-77500fcff9eb: !Template - answer_choices: null - id: 53bb4f17-cc2c-40ee-b696-77500fcff9eb + 728f9769-e4aa-40e0-bce0-7570cb622caa: !Template + answer_choices: A ||| B + id: 728f9769-e4aa-40e0-bce0-7570cb622caa jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -108,11 +130,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: better_B_A reference: '' - 6ee20533-4e94-4fe3-bc31-d9664905faeb: !Template - answer_choices: null - id: 6ee20533-4e94-4fe3-bc31-d9664905faeb + aef524e8-5ff7-4dd0-bcab-015fb33a2818: !Template + answer_choices: A ||| B + id: aef524e8-5ff7-4dd0-bcab-015fb33a2818 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer A or B. @@ -135,11 +157,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -147,31 +169,31 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: grammatical_between_A_B reference: '' - a694e34c-2731-492d-ab18-63575ec8befb: !Template + b4f8537b-64f3-48ff-b445-3d9cf3e1b285: !Template answer_choices: Yes ||| No - id: a694e34c-2731-492d-ab18-63575ec8befb + id: b4f8537b-64f3-48ff-b445-3d9cf3e1b285 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' + name: single_sentence_bad_yes_no reference: '' - d53c294d-ca94-4e3c-97d6-917acada0754: !Template + c39e11fa-c69f-4fb1-b24a-ac154ddc0667: !Template answer_choices: Yes ||| No - id: d53c294d-ca94-4e3c-97d6-917acada0754 + id: c39e11fa-c69f-4fb1-b24a-ac154ddc0667 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata @@ -179,26 +201,26 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' + name: single_sentence_bad_no_yes reference: '' - dd4a46eb-ffd5-4e9a-8068-245ff68a3e7b: !Template - answer_choices: null - id: dd4a46eb-ffd5-4e9a-8068-245ff68a3e7b + c494a50d-45e9-4ece-a9fb-869930283074: !Template + answer_choices: 1 ||| 2 + id: c494a50d-45e9-4ece-a9fb-869930283074 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -206,11 +228,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -218,11 +240,27 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: grammatical_between_1_2 + reference: '' + c8b8a68e-d91e-4db8-9698-4c6aa15db7e6: !Template + answer_choices: Yes ||| No + id: c8b8a68e-d91e-4db8-9698-4c6aa15db7e6 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no reference: '' - f0179b4a-9fdb-430c-928c-8accc7ba42cf: !Template - answer_choices: null - id: f0179b4a-9fdb-430c-928c-8accc7ba42cf + ddf494f1-47ae-4c4a-a726-8b9ff4bfff04: !Template + answer_choices: 1 ||| 2 + id: ddf494f1-47ae-4c4a-a726-8b9ff4bfff04 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer 1 or 2. @@ -245,44 +283,6 @@ templates: {% if shuffled_order == 0 %} - 1 - - {% else %} - - 2 - - {% endif %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: random_choice_1_2 - reference: '' - f9cb8868-cfea-4558-9e94-c66923e3b110: !Template - answer_choices: A ||| B - id: f9cb8868-cfea-4558-9e94-c66923e3b110 - jinja: '{% set shuffled_order = [0, 1] | random %} - - Read the following two sentences. - - {% if shuffled_order == 0 %} - - Sentence A: {{ sentence_good }} - - Sentence B: {{ sentence_bad }} - - {% else %} - - Sentence A: {{ sentence_bad }} - - Sentence B: {{ sentence_good }} - - {% endif %} - - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} - {{ answer_choices[0] }} {% else %} @@ -295,5 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: A-B)' + name: grammatical_which_one_1_2 reference: '' diff --git a/promptsource/templates/blimp/wh_questions_subject_gap_long_distance/templates.yaml b/promptsource/templates/blimp/wh_questions_subject_gap_long_distance/templates.yaml index 3eb4a2ebe..8f028bbb5 100644 --- a/promptsource/templates/blimp/wh_questions_subject_gap_long_distance/templates.yaml +++ b/promptsource/templates/blimp/wh_questions_subject_gap_long_distance/templates.yaml @@ -1,24 +1,56 @@ dataset: blimp subset: wh_questions_subject_gap_long_distance templates: - 4a4adcfb-18f9-4ad0-8272-ddd5ce2c02dd: !Template - answer_choices: null - id: 4a4adcfb-18f9-4ad0-8272-ddd5ce2c02dd + 28ebe131-6988-4bb0-a157-a0f7accef5c8: !Template + answer_choices: Yes ||| No + id: 28ebe131-6988-4bb0-a157-a0f7accef5c8 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 4eaf3a66-94d5-4e78-a0e0-fbf73ac9fccd: !Template + answer_choices: Yes ||| No + id: 4eaf3a66-94d5-4e78-a0e0-fbf73ac9fccd + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes + reference: '' + 4eba01cc-3f89-4ee2-9847-b4078b71a224: !Template + answer_choices: 1 ||| 2 + id: 4eba01cc-3f89-4ee2-9847-b4078b71a224 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -26,11 +58,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -38,26 +70,26 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: grammatical_between_1_2 reference: '' - 4d15b6c5-c750-4a18-9c92-f9d339e4e78d: !Template - answer_choices: null - id: 4d15b6c5-c750-4a18-9c92-f9d339e4e78d + 5b04b4a2-11ad-4976-8e7c-dae6f15651a8: !Template + answer_choices: 1 ||| 2 + id: 5b04b4a2-11ad-4976-8e7c-dae6f15651a8 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -65,11 +97,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -77,31 +109,32 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: grammatical_which_one_1_2 reference: '' - 56196cd7-6e97-464b-bb60-11e901c32f42: !Template + 8ce39c09-68b5-4356-9a5d-50f698776bdf: !Template answer_choices: A ||| B - id: 56196cd7-6e97-464b-bb60-11e901c32f42 + id: 8ce39c09-68b5-4356-9a5d-50f698776bdf jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -115,38 +148,37 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: A-B)' + name: grammatical_which_one_A_B reference: '' - 5c73149e-6698-40dd-b76b-2ec5da7cfa2b: !Template - answer_choices: null - id: 5c73149e-6698-40dd-b76b-2ec5da7cfa2b + b6f677a8-d25b-4c30-a818-9e7abe2edca8: !Template + answer_choices: A ||| B + id: b6f677a8-d25b-4c30-a818-9e7abe2edca8 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -154,27 +186,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 - reference: '' - a7b5726a-9c85-4a67-9064-7510093db2bf: !Template - answer_choices: Yes ||| No - id: a7b5726a-9c85-4a67-9064-7510093db2bf - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' + name: better_B_A reference: '' - c4e3a884-1219-4fd1-92cc-1da52d48b934: !Template + c7607b5c-2383-41de-8e05-5a3975d5be2d: !Template answer_choices: Yes ||| No - id: c4e3a884-1219-4fd1-92cc-1da52d48b934 + id: c7607b5c-2383-41de-8e05-5a3975d5be2d jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -186,42 +202,26 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' - reference: '' - d2ca0f0e-586f-4476-a1c0-41b1a2faf47a: !Template - answer_choices: Yes ||| No - id: d2ca0f0e-586f-4476-a1c0-41b1a2faf47a - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' + name: single_sentence_bad_no_yes reference: '' - d39e16be-1272-4358-8ab1-3bb4e80bc87b: !Template - answer_choices: null - id: d39e16be-1272-4358-8ab1-3bb4e80bc87b + d385fbf9-6bbb-4d9f-96e3-b3e039d6679d: !Template + answer_choices: A ||| B + id: d385fbf9-6bbb-4d9f-96e3-b3e039d6679d jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -229,11 +229,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -241,11 +241,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: grammatical_between_A_B reference: '' - d7692403-42a6-4b12-bea7-13b3abf271c0: !Template + ecab1d1c-7f5a-488b-b2a9-f8d233c1d832: !Template answer_choices: A ||| B - id: d7692403-42a6-4b12-bea7-13b3abf271c0 + id: ecab1d1c-7f5a-488b-b2a9-f8d233c1d832 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -264,7 +264,7 @@ templates: {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order + Which one is a better sentence of English, A or B? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -279,11 +279,11 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: B-A)' + name: better_A_B reference: '' - e07ea319-9726-4e78-ac86-aa18cb45ade4: !Template + f477c93b-9cd6-4504-8dc4-0e9e871a8c8c: !Template answer_choices: Yes ||| No - id: e07ea319-9726-4e78-ac86-aa18cb45ade4 + id: f477c93b-9cd6-4504-8dc4-0e9e871a8c8c jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -295,5 +295,5 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' + name: single_sentence_bad_yes_no reference: '' diff --git a/promptsource/templates/blimp/wh_vs_that_no_gap/templates.yaml b/promptsource/templates/blimp/wh_vs_that_no_gap/templates.yaml index 6cd1a4c1d..b891cd408 100644 --- a/promptsource/templates/blimp/wh_vs_that_no_gap/templates.yaml +++ b/promptsource/templates/blimp/wh_vs_that_no_gap/templates.yaml @@ -1,9 +1,9 @@ dataset: blimp subset: wh_vs_that_no_gap templates: - 08741ec2-b18b-48ea-9093-a7abdfe263e1: !Template + 02fe381d-25e7-42c9-8cba-05ef1c299ea5: !Template answer_choices: Yes ||| No - id: 08741ec2-b18b-48ea-9093-a7abdfe263e1 + id: 02fe381d-25e7-42c9-8cba-05ef1c299ea5 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -15,26 +15,26 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' + name: single_sentence_good_yes_no reference: '' - 126eb196-b508-4ec9-9b0a-febb0896d683: !Template - answer_choices: null - id: 126eb196-b508-4ec9-9b0a-febb0896d683 + 1c9d0f71-728e-470c-940e-20eca46eb36e: !Template + answer_choices: A ||| B + id: 1c9d0f71-728e-470c-940e-20eca46eb36e jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -42,11 +42,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -54,31 +54,32 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: grammatical_between_A_B reference: '' - 598ac8e0-c837-4394-9e63-7aae2b17a742: !Template - answer_choices: A ||| B - id: 598ac8e0-c837-4394-9e63-7aae2b17a742 + 2a5149f4-35e0-471b-92e5-41a8f6c12e6b: !Template + answer_choices: 1 ||| 2 + id: 2a5149f4-35e0-471b-92e5-41a8f6c12e6b jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -92,11 +93,11 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: A-B)' + name: grammatical_between_1_2 reference: '' - 83ee5881-9a7d-430b-ac8f-cd4b425729ad: !Template + 4f35487d-2f76-41ca-88cd-ec1771800dc7: !Template answer_choices: A ||| B - id: 83ee5881-9a7d-430b-ac8f-cd4b425729ad + id: 4f35487d-2f76-41ca-88cd-ec1771800dc7 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -115,7 +116,7 @@ templates: {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order + Which one is a better sentence of English, A or B? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -130,50 +131,11 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: B-A)' - reference: '' - 90e9e76b-791a-4c5e-89b6-ffb13061ca98: !Template - answer_choices: null - id: 90e9e76b-791a-4c5e-89b6-ffb13061ca98 - jinja: '{% set shuffled_order = [0, 1] | random %} - - Which one of the following sentences is grammatical? Please answer 1 or 2. - - {% if shuffled_order == 0 %} - - 1: {{ sentence_good }} - - 2: {{ sentence_bad }} - - {% else %} - - 1: {{ sentence_bad }} - - 2: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - 1 - - {% else %} - - 2 - - {% endif %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: random_choice_1_2 + name: better_A_B reference: '' - 9fed0c48-2413-499f-9b7b-b42d3a425d92: !Template + 6c800564-f349-4328-a858-9982c8373a11: !Template answer_choices: Yes ||| No - id: 9fed0c48-2413-499f-9b7b-b42d3a425d92 + id: 6c800564-f349-4328-a858-9982c8373a11 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -185,38 +147,37 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' + name: single_sentence_bad_yes_no reference: '' - bd3ed699-075b-40f1-9b8e-bb28179645dd: !Template - answer_choices: null - id: bd3ed699-075b-40f1-9b8e-bb28179645dd + 723c2dcf-9f76-401e-b0b7-d6c19db0900f: !Template + answer_choices: A ||| B + id: 723c2dcf-9f76-401e-b0b7-d6c19db0900f jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -224,11 +185,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: better_B_A reference: '' - bdb81024-b38f-441d-9533-96ea05dc9faa: !Template - answer_choices: null - id: bdb81024-b38f-441d-9533-96ea05dc9faa + 932625dd-b96e-4d24-bdff-abcf59dbeead: !Template + answer_choices: A ||| B + id: 932625dd-b96e-4d24-bdff-abcf59dbeead jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer A or B. @@ -251,11 +212,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -263,37 +224,76 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: grammatical_which_one_A_B reference: '' - d3399774-0cec-4c85-8348-2a78eff8508e: !Template + bd7515c0-8c86-4e76-b456-bd4a0b9b9e46: !Template answer_choices: Yes ||| No - id: d3399774-0cec-4c85-8348-2a78eff8508e + id: bd7515c0-8c86-4e76-b456-bd4a0b9b9e46 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' + name: single_sentence_bad_no_yes reference: '' - eb12dd1f-efcd-4835-aaff-96fe44bab9b2: !Template + da4987d5-8a06-4e72-8381-ef17f3c21133: !Template answer_choices: Yes ||| No - id: eb12dd1f-efcd-4835-aaff-96fe44bab9b2 + id: da4987d5-8a06-4e72-8381-ef17f3c21133 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' + name: single_sentence_good_no_yes + reference: '' + e5c16d1f-d8eb-4f79-b027-a450f9776ed4: !Template + answer_choices: 1 ||| 2 + id: e5c16d1f-d8eb-4f79-b027-a450f9776ed4 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_1_2 reference: '' diff --git a/promptsource/templates/blimp/wh_vs_that_no_gap_long_distance/templates.yaml b/promptsource/templates/blimp/wh_vs_that_no_gap_long_distance/templates.yaml index 73bb92cc9..a4f7e685e 100644 --- a/promptsource/templates/blimp/wh_vs_that_no_gap_long_distance/templates.yaml +++ b/promptsource/templates/blimp/wh_vs_that_no_gap_long_distance/templates.yaml @@ -1,48 +1,9 @@ dataset: blimp subset: wh_vs_that_no_gap_long_distance templates: - 24d4f457-c24e-462d-a5eb-9cce58b17b1b: !Template - answer_choices: null - id: 24d4f457-c24e-462d-a5eb-9cce58b17b1b - jinja: '{% set shuffled_order = [0, 1] | random %} - - Between the two sentences, which one is grammatical? Please answer 1 or 2. - - {% if shuffled_order == 0 %} - - 1: {{ sentence_good }} - - 2: {{ sentence_bad }} - - {% else %} - - 1: {{ sentence_bad }} - - 2: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - 1 - - {% else %} - - 2 - - {% endif %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: random_choice_1_2_2 - reference: '' - 261d37b8-c0f5-4ab3-bad6-b914fab9f748: !Template + 090af836-a282-47a4-82b8-1b33708cdc55: !Template answer_choices: A ||| B - id: 261d37b8-c0f5-4ab3-bad6-b914fab9f748 + id: 090af836-a282-47a4-82b8-1b33708cdc55 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -76,15 +37,15 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: B-A)' + name: better_B_A reference: '' - 2da6237a-18fe-4f35-837b-983bb0bd8ecb: !Template + 2fffb3ed-4be4-4bac-a847-0296482de5f1: !Template answer_choices: Yes ||| No - id: 2da6237a-18fe-4f35-837b-983bb0bd8ecb + id: 2fffb3ed-4be4-4bac-a847-0296482de5f1 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata @@ -92,15 +53,15 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' + name: single_sentence_bad_yes_no reference: '' - 338fb184-4c96-4557-9916-0989d2e71027: !Template + 5105a5c0-e61c-49ba-b8ff-24b5260c51f9: !Template answer_choices: Yes ||| No - id: 338fb184-4c96-4557-9916-0989d2e71027 + id: 5105a5c0-e61c-49ba-b8ff-24b5260c51f9 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -108,15 +69,31 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' + name: single_sentence_good_yes_no reference: '' - 4b0e8f53-6e0f-402c-93f4-d59a44593a47: !Template + 59bf12ae-b890-42aa-8c15-8bd483732eed: !Template answer_choices: Yes ||| No - id: 4b0e8f53-6e0f-402c-93f4-d59a44593a47 + id: 59bf12ae-b890-42aa-8c15-8bd483732eed jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes + reference: '' + 612e1ab2-7ec8-4f52-8e2e-6f3490aff846: !Template + answer_choices: Yes ||| No + id: 612e1ab2-7ec8-4f52-8e2e-6f3490aff846 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -124,26 +101,26 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' + name: single_sentence_good_no_yes reference: '' - 76821051-fc90-4b78-8717-f886ddded5da: !Template - answer_choices: null - id: 76821051-fc90-4b78-8717-f886ddded5da + 6a8803b9-b898-423c-80f6-9dcf587edca7: !Template + answer_choices: 1 ||| 2 + id: 6a8803b9-b898-423c-80f6-9dcf587edca7 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -151,11 +128,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -163,27 +140,50 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: grammatical_between_1_2 reference: '' - 9b6e51cc-079f-44ee-bce9-7558da58e11f: !Template - answer_choices: Yes ||| No - id: 9b6e51cc-079f-44ee-bce9-7558da58e11f - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + a69a6d39-2b8b-4ae4-b7e6-a7e4408be701: !Template + answer_choices: A ||| B + id: a69a6d39-2b8b-4ae4-b7e6-a7e4408be701 + jinja: '{% set shuffled_order = [0, 1] | random %} - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' + original_task: true + name: grammatical_which_one_A_B reference: '' - aec7db7f-1eb7-4247-ac42-4c3a429f36aa: !Template + af64e812-5ab7-4d8f-ab82-7f5f8fdf20db: !Template answer_choices: A ||| B - id: aec7db7f-1eb7-4247-ac42-4c3a429f36aa + id: af64e812-5ab7-4d8f-ab82-7f5f8fdf20db jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -217,11 +217,11 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: A-B)' + name: better_A_B reference: '' - d36cfd2a-8c25-4c74-80ee-5bc90a6f7b00: !Template - answer_choices: null - id: d36cfd2a-8c25-4c74-80ee-5bc90a6f7b00 + b5b8e78f-9ef8-4bf2-b4c3-d14fa424ead1: !Template + answer_choices: A ||| B + id: b5b8e78f-9ef8-4bf2-b4c3-d14fa424ead1 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer A or B. @@ -244,11 +244,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -256,11 +256,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: grammatical_between_A_B reference: '' - e7ad406a-164c-4662-8341-b9febd044954: !Template - answer_choices: null - id: e7ad406a-164c-4662-8341-b9febd044954 + ff19dd12-be84-4286-b666-1a00a145ac80: !Template + answer_choices: 1 ||| 2 + id: ff19dd12-be84-4286-b666-1a00a145ac80 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer 1 or 2. @@ -283,11 +283,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -295,5 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: grammatical_which_one_1_2 reference: '' diff --git a/promptsource/templates/blimp/wh_vs_that_with_gap/templates.yaml b/promptsource/templates/blimp/wh_vs_that_with_gap/templates.yaml index a8dacbace..da5423ce2 100644 --- a/promptsource/templates/blimp/wh_vs_that_with_gap/templates.yaml +++ b/promptsource/templates/blimp/wh_vs_that_with_gap/templates.yaml @@ -1,40 +1,40 @@ dataset: blimp subset: wh_vs_that_with_gap templates: - 30d16980-86aa-47da-999d-eea0f4808ebb: !Template + 1a2a0afd-ac1a-4969-bc21-7f2e1dde361a: !Template answer_choices: Yes ||| No - id: 30d16980-86aa-47da-999d-eea0f4808ebb + id: 1a2a0afd-ac1a-4969-bc21-7f2e1dde361a jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' + name: single_sentence_bad_no_yes reference: '' - 88cfbc9a-7bd8-4a26-aaeb-68216eab7628: !Template - answer_choices: null - id: 88cfbc9a-7bd8-4a26-aaeb-68216eab7628 + 345bd841-7af7-4381-9a44-ab9497e5c7ce: !Template + answer_choices: 1 ||| 2 + id: 345bd841-7af7-4381-9a44-ab9497e5c7ce jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -42,11 +42,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -54,27 +54,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B - reference: '' - 9200c8a2-206d-4682-8d83-91f1b7e5c180: !Template - answer_choices: Yes ||| No - id: 9200c8a2-206d-4682-8d83-91f1b7e5c180 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' + name: grammatical_between_1_2 reference: '' - b6f46e80-e359-48b1-ae2e-66b1385683f5: !Template - answer_choices: null - id: b6f46e80-e359-48b1-ae2e-66b1385683f5 + 37019fb3-2423-4f6e-b6c9-12ef27ea0395: !Template + answer_choices: A ||| B + id: 37019fb3-2423-4f6e-b6c9-12ef27ea0395 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer A or B. @@ -97,44 +81,6 @@ templates: {% if shuffled_order == 0 %} - A - - {% else %} - - B - - {% endif %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: random_choice_A_B_2 - reference: '' - c4397157-910e-493c-9482-55996ccd712c: !Template - answer_choices: A ||| B - id: c4397157-910e-493c-9482-55996ccd712c - jinja: '{% set shuffled_order = [0, 1] | random %} - - Read the following two sentences. - - {% if shuffled_order == 0 %} - - Sentence A: {{ sentence_good }} - - Sentence B: {{ sentence_bad }} - - {% else %} - - Sentence A: {{ sentence_bad }} - - Sentence B: {{ sentence_good }} - - {% endif %} - - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} - {{ answer_choices[0] }} {% else %} @@ -147,27 +93,27 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: B-A)' + name: grammatical_between_A_B reference: '' - c9b845cf-7b73-430c-81f4-c27286d6e2a0: !Template + 4147d3d1-b692-434f-8d06-f6ca82edb8f3: !Template answer_choices: Yes ||| No - id: c9b845cf-7b73-430c-81f4-c27286d6e2a0 + id: 4147d3d1-b692-434f-8d06-f6ca82edb8f3 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' + name: single_sentence_good_yes_no reference: '' - cde97831-7118-4cdd-a28e-b369cb1cc6a0: !Template + 63005ef4-90e4-4d0e-bd82-54e82eb9118a: !Template answer_choices: A ||| B - id: cde97831-7118-4cdd-a28e-b369cb1cc6a0 + id: 63005ef4-90e4-4d0e-bd82-54e82eb9118a jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -201,14 +147,14 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: A-B)' + name: better_A_B reference: '' - d0dc04f5-e132-47f0-b264-ee44815a417b: !Template - answer_choices: null - id: d0dc04f5-e132-47f0-b264-ee44815a417b + 77255ece-e099-42db-a582-75477f76f5f7: !Template + answer_choices: 1 ||| 2 + id: 77255ece-e099-42db-a582-75477f76f5f7 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -228,11 +174,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -240,38 +186,53 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: grammatical_which_one_1_2 reference: '' - d919cd6a-6031-4835-a7a5-f61602cda175: !Template - answer_choices: null - id: d919cd6a-6031-4835-a7a5-f61602cda175 + e92eb612-9467-4be7-aeee-98b8b1772895: !Template + answer_choices: Yes ||| No + id: e92eb612-9467-4be7-aeee-98b8b1772895 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes + reference: '' + ecb09615-c2a0-4292-8693-e5ed817e230e: !Template + answer_choices: A ||| B + id: ecb09615-c2a0-4292-8693-e5ed817e230e jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -279,21 +240,60 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: better_B_A reference: '' - e2ce39cf-1e36-4a2f-b76e-ba40473d4109: !Template + ecdeeef8-1672-4dc4-8963-eefb74dc2a5e: !Template answer_choices: Yes ||| No - id: e2ce39cf-1e36-4a2f-b76e-ba40473d4109 + id: ecdeeef8-1672-4dc4-8963-eefb74dc2a5e jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' + name: single_sentence_bad_yes_no + reference: '' + f53a2291-d705-401a-b4c6-1f02b3fb960c: !Template + answer_choices: A ||| B + id: f53a2291-d705-401a-b4c6-1f02b3fb960c + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_A_B reference: '' diff --git a/promptsource/templates/blimp/wh_vs_that_with_gap_long_distance/templates.yaml b/promptsource/templates/blimp/wh_vs_that_with_gap_long_distance/templates.yaml index b5e0c7b92..5d6162245 100644 --- a/promptsource/templates/blimp/wh_vs_that_with_gap_long_distance/templates.yaml +++ b/promptsource/templates/blimp/wh_vs_that_with_gap_long_distance/templates.yaml @@ -1,47 +1,9 @@ dataset: blimp subset: wh_vs_that_with_gap_long_distance templates: - 03f877b2-092b-49f2-bad1-80b7a136e720: !Template - answer_choices: A ||| B - id: 03f877b2-092b-49f2-bad1-80b7a136e720 - jinja: '{% set shuffled_order = [0, 1] | random %} - - Read the following two sentences. - - {% if shuffled_order == 0 %} - - Sentence A: {{ sentence_good }} - - Sentence B: {{ sentence_bad }} - - {% else %} - - Sentence A: {{ sentence_bad }} - - Sentence B: {{ sentence_good }} - - {% endif %} - - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} - - {% endif %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: 'A/B choice randomized (choice order: A-B)' - reference: '' - 044612a2-d57b-4534-9eb1-3f6fdf902294: !Template - answer_choices: null - id: 044612a2-d57b-4534-9eb1-3f6fdf902294 + 0b9eb74c-c82e-4709-84b0-85dc9bf6eace: !Template + answer_choices: 1 ||| 2 + id: 0b9eb74c-c82e-4709-84b0-85dc9bf6eace jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -64,11 +26,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -76,38 +38,53 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2_2 + name: grammatical_between_1_2 reference: '' - 19780724-c516-4229-bbc4-53f6055e1076: !Template - answer_choices: null - id: 19780724-c516-4229-bbc4-53f6055e1076 + 0e3396fc-9559-4e9d-8ee4-254d55f11710: !Template + answer_choices: Yes ||| No + id: 0e3396fc-9559-4e9d-8ee4-254d55f11710 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes + reference: '' + 1edac537-702e-4086-8ce0-fc0723ab34e2: !Template + answer_choices: A ||| B + id: 1edac537-702e-4086-8ce0-fc0723ab34e2 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -115,26 +92,26 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B + name: better_A_B reference: '' - 2c516153-a176-4a03-a109-1c7d5d508872: !Template - answer_choices: null - id: 2c516153-a176-4a03-a109-1c7d5d508872 + 32853a41-c3fe-4a80-bcaf-eb6be1e7ebc9: !Template + answer_choices: A ||| B + id: 32853a41-c3fe-4a80-bcaf-eb6be1e7ebc9 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -142,11 +119,11 @@ templates: {% if shuffled_order == 0 %} - 1 + {{ answer_choices[0] }} {% else %} - 2 + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -154,11 +131,11 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_1_2 + name: grammatical_between_A_B reference: '' - 358d3f20-8d63-402c-9cf7-d28df1aa4e77: !Template + 458ea68a-0659-40dc-ac9a-478cb828cc1f: !Template answer_choices: A ||| B - id: 358d3f20-8d63-402c-9cf7-d28df1aa4e77 + id: 458ea68a-0659-40dc-ac9a-478cb828cc1f jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -192,15 +169,15 @@ templates: metrics: - Accuracy original_task: true - name: 'A/B choice randomized (choice order: B-A)' + name: better_B_A reference: '' - 3de137a6-a2d5-4bb5-b31a-7e2cc7a08828: !Template + bb086356-5183-46c8-aab7-dc28e814c35d: !Template answer_choices: Yes ||| No - id: 3de137a6-a2d5-4bb5-b31a-7e2cc7a08828 + id: bb086356-5183-46c8-aab7-dc28e814c35d jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata @@ -208,15 +185,15 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: No-Yes)' + name: single_sentence_bad_yes_no reference: '' - e73651a0-d884-45b3-8ba4-c26a2c2c2051: !Template + bc533a50-f176-4c73-8cc7-2ab0819bc641: !Template answer_choices: Yes ||| No - id: e73651a0-d884-45b3-8ba4-c26a2c2c2051 + id: bc533a50-f176-4c73-8cc7-2ab0819bc641 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -224,15 +201,15 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: No-Yes)' + name: single_sentence_good_yes_no reference: '' - f1764f9f-d8c5-49f0-8100-9b344edefef6: !Template + c9f796cb-e35d-4128-8d2a-5fdd6c86f24f: !Template answer_choices: Yes ||| No - id: f1764f9f-d8c5-49f0-8100-9b344edefef6 + id: c9f796cb-e35d-4128-8d2a-5fdd6c86f24f jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata @@ -240,42 +217,65 @@ templates: metrics: - Accuracy original_task: false - name: 'Yes/No binary judgment for bad sentences (choice order: Yes-No)' + name: single_sentence_bad_no_yes reference: '' - f762d6f8-3685-4614-9ac2-7b2eeae93415: !Template - answer_choices: Yes ||| No - id: f762d6f8-3685-4614-9ac2-7b2eeae93415 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + d36bdeb7-6316-4dd5-a029-9db2ecd3ef11: !Template + answer_choices: A ||| B + id: d36bdeb7-6316-4dd5-a029-9db2ecd3ef11 + jinja: '{% set shuffled_order = [0, 1] | random %} - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: false - name: 'Yes/No binary judgment for good sentences (choice order: Yes-No)' + original_task: true + name: grammatical_which_one_A_B reference: '' - fa4b90ca-133a-4712-bc83-c3280b5518a1: !Template - answer_choices: null - id: fa4b90ca-133a-4712-bc83-c3280b5518a1 + eff28ba0-08ec-4149-bbe6-c3a9a6fab62a: !Template + answer_choices: 1 ||| 2 + id: eff28ba0-08ec-4149-bbe6-c3a9a6fab62a jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -283,11 +283,11 @@ templates: {% if shuffled_order == 0 %} - A + {{ answer_choices[0] }} {% else %} - B + {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -295,5 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: random_choice_A_B_2 + name: grammatical_which_one_1_2 reference: '' From e91b438c81c12924a5c0b9b8d5e4b82a10590fb8 Mon Sep 17 00:00:00 2001 From: najoungkim Date: Tue, 26 Apr 2022 17:56:32 -0400 Subject: [PATCH 13/16] Fixed extra spaces. --- .../blimp/adjunct_island/templates.yaml | 8 +- .../anaphor_gender_agreement/templates.yaml | 222 +++++++++--------- .../anaphor_number_agreement/templates.yaml | 208 ++++++++-------- .../animate_subject_passive/templates.yaml | 178 +++++++------- .../animate_subject_trans/templates.yaml | 186 +++++++-------- .../templates/blimp/causative/templates.yaml | 176 +++++++------- .../blimp/complex_NP_island/templates.yaml | 186 +++++++-------- .../templates.yaml | 182 +++++++------- .../templates.yaml | 200 ++++++++-------- .../templates.yaml | 134 +++++------ .../templates.yaml | 202 ++++++++-------- .../templates.yaml | 186 +++++++-------- .../templates.yaml | 202 ++++++++-------- .../templates.yaml | 166 ++++++------- .../templates.yaml | 176 +++++++------- .../templates.yaml | 166 ++++++------- .../templates.yaml | 210 ++++++++--------- .../templates.yaml | 162 ++++++------- .../templates.yaml | 194 +++++++-------- .../blimp/drop_argument/templates.yaml | 148 ++++++------ .../blimp/ellipsis_n_bar_1/templates.yaml | 176 +++++++------- .../blimp/ellipsis_n_bar_2/templates.yaml | 146 ++++++------ .../templates.yaml | 202 ++++++++-------- .../templates.yaml | 160 ++++++------- .../templates.yaml | 200 ++++++++-------- .../templates.yaml | 212 ++++++++--------- .../templates.yaml | 182 +++++++------- .../templates/blimp/inchoative/templates.yaml | 162 ++++++------- .../blimp/intransitive/templates.yaml | 192 +++++++-------- .../templates.yaml | 188 +++++++-------- .../templates.yaml | 142 +++++------ .../templates.yaml | 138 +++++------ .../templates.yaml | 206 ++++++++-------- .../templates.yaml | 166 ++++++------- .../templates.yaml | 206 ++++++++-------- .../templates.yaml | 196 ++++++++-------- .../blimp/npi_present_1/templates.yaml | 180 +++++++------- .../blimp/npi_present_2/templates.yaml | 194 +++++++-------- .../only_npi_licensor_present/templates.yaml | 204 ++++++++-------- .../blimp/only_npi_scope/templates.yaml | 208 ++++++++-------- .../templates/blimp/passive_1/templates.yaml | 166 ++++++------- .../templates/blimp/passive_2/templates.yaml | 186 +++++++-------- .../principle_A_c_command/templates.yaml | 196 ++++++++-------- .../blimp/principle_A_case_1/templates.yaml | 196 ++++++++-------- .../blimp/principle_A_case_2/templates.yaml | 180 +++++++------- .../blimp/principle_A_domain_1/templates.yaml | 202 ++++++++-------- .../blimp/principle_A_domain_2/templates.yaml | 198 ++++++++-------- .../blimp/principle_A_domain_3/templates.yaml | 178 +++++++------- .../principle_A_reconstruction/templates.yaml | 170 +++++++------- .../templates.yaml | 214 ++++++++--------- .../templates.yaml | 170 +++++++------- .../templates.yaml | 196 ++++++++-------- .../templates.yaml | 160 ++++++------- .../sentential_subject_island/templates.yaml | 212 ++++++++--------- .../superlative_quantifiers_1/templates.yaml | 184 +++++++-------- .../superlative_quantifiers_2/templates.yaml | 154 ++++++------ .../blimp/tough_vs_raising_1/templates.yaml | 196 ++++++++-------- .../blimp/tough_vs_raising_2/templates.yaml | 154 ++++++------ 58 files changed, 5232 insertions(+), 5232 deletions(-) diff --git a/promptsource/templates/blimp/adjunct_island/templates.yaml b/promptsource/templates/blimp/adjunct_island/templates.yaml index 166acc487..8bb424bc2 100644 --- a/promptsource/templates/blimp/adjunct_island/templates.yaml +++ b/promptsource/templates/blimp/adjunct_island/templates.yaml @@ -10,13 +10,13 @@ templates: {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + Sentence A: {{ sentence_good }} Sentence B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} Sentence B: {{ sentence_good }} @@ -213,13 +213,13 @@ templates: {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + Sentence A: {{ sentence_good }} Sentence B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} Sentence B: {{ sentence_good }} diff --git a/promptsource/templates/blimp/anaphor_gender_agreement/templates.yaml b/promptsource/templates/blimp/anaphor_gender_agreement/templates.yaml index 0cd1085fa..0f904a6c3 100644 --- a/promptsource/templates/blimp/anaphor_gender_agreement/templates.yaml +++ b/promptsource/templates/blimp/anaphor_gender_agreement/templates.yaml @@ -1,9 +1,80 @@ dataset: blimp subset: anaphor_gender_agreement templates: - 2ce6d65f-1230-47ee-8900-685880bce0c3: !Template + 14410b9a-0ae9-4e49-b2ec-cdce6a8fba37: !Template + answer_choices: A ||| B + id: 14410b9a-0ae9-4e49-b2ec-cdce6a8fba37 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_A_B + reference: '' + 27066b83-d53e-4260-ac95-e4ba390c78c3: !Template + answer_choices: Yes ||| No + id: 27066b83-d53e-4260-ac95-e4ba390c78c3 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes + reference: '' + 289dda3f-3052-4851-bd8c-d53a71355a3c: !Template + answer_choices: Yes ||| No + id: 289dda3f-3052-4851-bd8c-d53a71355a3c + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes + reference: '' + 302c2622-e23b-422b-8821-6bfa410e0038: !Template answer_choices: 1 ||| 2 - id: 2ce6d65f-1230-47ee-8900-685880bce0c3 + id: 302c2622-e23b-422b-8821-6bfa410e0038 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -40,28 +111,28 @@ templates: original_task: true name: grammatical_between_1_2 reference: '' - 2e1d6529-58d2-454d-a1fe-49ddec83b3fd: !Template + 38a12baf-18de-4fac-8689-3e2c5fcdbd74: !Template answer_choices: A ||| B - id: 2e1d6529-58d2-454d-a1fe-49ddec83b3fd + id: 38a12baf-18de-4fac-8689-3e2c5fcdbd74 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + Sentence A: {{ sentence_good }} Sentence B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} Sentence B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order + Which one is a better sentence of English, B or A? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -76,46 +147,14 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B - reference: '' - 36a58157-aa61-421d-ba18-053a2c3f3456: !Template - answer_choices: Yes ||| No - id: 36a58157-aa61-421d-ba18-053a2c3f3456 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no - reference: '' - 4e72a002-ae42-4a95-b390-b008922ebd45: !Template - answer_choices: Yes ||| No - id: 4e72a002-ae42-4a95-b390-b008922ebd45 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes + name: better_B_A reference: '' - 5380667f-a977-42ac-b00b-a69ee9f8f8fd: !Template + 4abf6023-679a-4f9f-9a21-0ddb167b7bbd: !Template answer_choices: A ||| B - id: 5380667f-a977-42ac-b00b-a69ee9f8f8fd + id: 4abf6023-679a-4f9f-9a21-0ddb167b7bbd jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -147,15 +186,15 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: grammatical_which_one_A_B reference: '' - 57368d1e-d9e9-4e23-b6ca-a73f816c2036: !Template + 5da96864-24ae-44a2-93b5-9e4f4734d5de: !Template answer_choices: Yes ||| No - id: 57368d1e-d9e9-4e23-b6ca-a73f816c2036 + id: 5da96864-24ae-44a2-93b5-9e4f4734d5de jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -163,11 +202,27 @@ templates: metrics: - Accuracy original_task: false - name: single_sentence_good_no_yes + name: single_sentence_good_yes_no + reference: '' + a3767823-ab46-4099-aad7-54a66b819210: !Template + answer_choices: Yes ||| No + id: a3767823-ab46-4099-aad7-54a66b819210 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no reference: '' - 5fab1923-c176-4e58-a2e0-238141d0b209: !Template + e111fc31-a73b-407d-997a-ba1c50cee471: !Template answer_choices: 1 ||| 2 - id: 5fab1923-c176-4e58-a2e0-238141d0b209 + id: e111fc31-a73b-407d-997a-ba1c50cee471 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer 1 or 2. @@ -204,28 +259,28 @@ templates: original_task: true name: grammatical_which_one_1_2 reference: '' - 6157d8d7-7889-4c4e-be23-2ce3c626ee4d: !Template + f209365f-af86-407a-bc9e-e609bcb8ae39: !Template answer_choices: A ||| B - id: 6157d8d7-7889-4c4e-be23-2ce3c626ee4d + id: f209365f-af86-407a-bc9e-e609bcb8ae39 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + Sentence A: {{ sentence_good }} Sentence B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} Sentence B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order + Which one is a better sentence of English, A or B? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -240,60 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A - reference: '' - 76ba2300-517d-406a-b07b-f0ff0eee62e4: !Template - answer_choices: Yes ||| No - id: 76ba2300-517d-406a-b07b-f0ff0eee62e4 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no - reference: '' - be4d8987-d4fb-4562-b99d-6625e2ab5f76: !Template - answer_choices: A ||| B - id: be4d8987-d4fb-4562-b99d-6625e2ab5f76 - jinja: '{% set shuffled_order = [0, 1] | random %} - - Which one of the following sentences is grammatical? Please answer A or B. - - {% if shuffled_order == 0 %} - - A: {{ sentence_good }} - - B: {{ sentence_bad }} - - {% else %} - - A: {{ sentence_bad }} - - B: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} - - {% endif %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: grammatical_which_one_A_B + name: better_A_B reference: '' diff --git a/promptsource/templates/blimp/anaphor_number_agreement/templates.yaml b/promptsource/templates/blimp/anaphor_number_agreement/templates.yaml index f7e2393d5..50e81f13a 100644 --- a/promptsource/templates/blimp/anaphor_number_agreement/templates.yaml +++ b/promptsource/templates/blimp/anaphor_number_agreement/templates.yaml @@ -1,24 +1,24 @@ dataset: blimp subset: anaphor_number_agreement templates: - 091d3d5c-a07f-4467-ba2f-d6b1923b22a1: !Template - answer_choices: 1 ||| 2 - id: 091d3d5c-a07f-4467-ba2f-d6b1923b22a1 + 1678bdf4-83cf-4427-9a26-420a114697d5: !Template + answer_choices: A ||| B + id: 1678bdf4-83cf-4427-9a26-420a114697d5 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -38,84 +38,46 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 - reference: '' - 128cee90-722e-4dde-91d2-3b4dade54971: !Template - answer_choices: A ||| B - id: 128cee90-722e-4dde-91d2-3b4dade54971 - jinja: '{% set shuffled_order = [0, 1] | random %} - - Read the following two sentences. - - {% if shuffled_order == 0 %} - - Sentence A: {{ sentence_good }} - - Sentence B: {{ sentence_bad }} - - {% else %} - - Sentence A: {{ sentence_bad }} - - Sentence B: {{ sentence_good }} - - {% endif %} - - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} - - {% endif %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: better_A_B + name: grammatical_which_one_A_B reference: '' - 19479faa-c653-456d-8267-f835593aa75f: !Template + 2a55a588-c014-425d-8c84-f4fadcc158d6: !Template answer_choices: Yes ||| No - id: 19479faa-c653-456d-8267-f835593aa75f + id: 2a55a588-c014-425d-8c84-f4fadcc158d6 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_bad_yes_no + name: single_sentence_good_yes_no reference: '' - 32bd7408-29f8-4a8e-ab14-940ac50cdae8: !Template + 2aa1369b-91a7-4c58-905b-f9ae34b8b9a1: !Template answer_choices: A ||| B - id: 32bd7408-29f8-4a8e-ab14-940ac50cdae8 + id: 2aa1369b-91a7-4c58-905b-f9ae34b8b9a1 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + Sentence A: {{ sentence_good }} Sentence B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} Sentence B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order + Which one is a better sentence of English, A or B? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -130,14 +92,14 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: better_A_B reference: '' - 6dadf8d0-4cff-4aa8-b54a-c4ed82f23cb2: !Template + 2e05b4df-bf2f-4e00-ab53-7ab454b0b325: !Template answer_choices: A ||| B - id: 6dadf8d0-4cff-4aa8-b54a-c4ed82f23cb2 + id: 2e05b4df-bf2f-4e00-ab53-7ab454b0b325 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -169,11 +131,27 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: grammatical_between_A_B + reference: '' + b2735fbf-80ea-4f88-adb2-70cedec7779c: !Template + answer_choices: Yes ||| No + id: b2735fbf-80ea-4f88-adb2-70cedec7779c + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no reference: '' - 7033308d-63b2-4f28-b0fa-2aec27de9e25: !Template + b780c5a5-c0e6-49dd-bb18-7d04ea6a4ed2: !Template answer_choices: 1 ||| 2 - id: 7033308d-63b2-4f28-b0fa-2aec27de9e25 + id: b780c5a5-c0e6-49dd-bb18-7d04ea6a4ed2 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -210,13 +188,29 @@ templates: original_task: true name: grammatical_between_1_2 reference: '' - 92dc5280-6832-4e8a-85bd-c5beb4bd5824: !Template + d58cd3c3-ec83-49df-8af5-ba2407dc9b98: !Template answer_choices: Yes ||| No - id: 92dc5280-6832-4e8a-85bd-c5beb4bd5824 + id: d58cd3c3-ec83-49df-8af5-ba2407dc9b98 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes + reference: '' + e645a293-3597-49d0-ac78-b5f6a8a20f39: !Template + answer_choices: Yes ||| No + id: e645a293-3597-49d0-ac78-b5f6a8a20f39 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -224,32 +218,31 @@ templates: metrics: - Accuracy original_task: false - name: single_sentence_good_yes_no + name: single_sentence_good_no_yes reference: '' - 9cd88feb-dc16-40c6-b642-ceb351edda27: !Template + e7cd3e1a-a079-478d-82c8-6c04552e63cf: !Template answer_choices: A ||| B - id: 9cd88feb-dc16-40c6-b642-ceb351edda27 + id: e7cd3e1a-a079-478d-82c8-6c04552e63cf jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -263,37 +256,44 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: better_B_A reference: '' - 9f6b4bab-73d8-45ce-8b30-b515c6a501cf: !Template - answer_choices: Yes ||| No - id: 9f6b4bab-73d8-45ce-8b30-b515c6a501cf - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + e9493f58-3944-4ed7-930b-75aeebe63d72: !Template + answer_choices: 1 ||| 2 + id: e9493f58-3944-4ed7-930b-75aeebe63d72 + jinja: '{% set shuffled_order = [0, 1] | random %} - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes - reference: '' - cab3fcac-1f71-4f62-b10c-2e8997ada36a: !Template - answer_choices: Yes ||| No - id: cab3fcac-1f71-4f62-b10c-2e8997ada36a - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + Which one of the following sentences is grammatical? Please answer 1 or 2. - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: false - name: single_sentence_bad_no_yes + original_task: true + name: grammatical_which_one_1_2 reference: '' diff --git a/promptsource/templates/blimp/animate_subject_passive/templates.yaml b/promptsource/templates/blimp/animate_subject_passive/templates.yaml index a97788a3f..fc8b37e87 100644 --- a/promptsource/templates/blimp/animate_subject_passive/templates.yaml +++ b/promptsource/templates/blimp/animate_subject_passive/templates.yaml @@ -1,9 +1,25 @@ dataset: blimp subset: animate_subject_passive templates: - 1742794e-41e7-4865-a7e4-fc9d3b56a491: !Template + 31f32f78-b308-45a6-bdfe-6855dcc6d8c6: !Template + answer_choices: Yes ||| No + id: 31f32f78-b308-45a6-bdfe-6855dcc6d8c6 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes + reference: '' + 5026b9ec-21d4-494d-80e1-2331080f44fd: !Template answer_choices: A ||| B - id: 1742794e-41e7-4865-a7e4-fc9d3b56a491 + id: 5026b9ec-21d4-494d-80e1-2331080f44fd jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer A or B. @@ -40,25 +56,9 @@ templates: original_task: true name: grammatical_between_A_B reference: '' - 4fec2ad9-0944-4cc8-9535-f089cf3ef14e: !Template + 6babaa62-0772-48db-80d4-c2cbc5671243: !Template answer_choices: Yes ||| No - id: 4fec2ad9-0944-4cc8-9535-f089cf3ef14e - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no - reference: '' - 59ee660e-a457-4325-813c-abcd6e40d15a: !Template - answer_choices: Yes ||| No - id: 59ee660e-a457-4325-813c-abcd6e40d15a + id: 6babaa62-0772-48db-80d4-c2cbc5671243 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -72,45 +72,30 @@ templates: original_task: false name: single_sentence_good_yes_no reference: '' - 6ed10f7d-da3e-41bb-8b1a-47489f7f5bca: !Template - answer_choices: Yes ||| No - id: 6ed10f7d-da3e-41bb-8b1a-47489f7f5bca - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes - reference: '' - 75146e21-4abe-4a21-98bf-72bb8c5458f0: !Template - answer_choices: A ||| B - id: 75146e21-4abe-4a21-98bf-72bb8c5458f0 + 88257b30-8c10-42f8-9df7-90a85431b575: !Template + answer_choices: 1 ||| 2 + id: 88257b30-8c10-42f8-9df7-90a85431b575 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -124,27 +109,11 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A - reference: '' - 76f0bb6b-ce88-4c2b-9b7c-37a3d744b65e: !Template - answer_choices: Yes ||| No - id: 76f0bb6b-ce88-4c2b-9b7c-37a3d744b65e - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes + name: grammatical_between_1_2 reference: '' - 82750934-e6fa-46b5-8d43-2cc53b3bcc9a: !Template + 8867366b-9956-4d75-a3fc-1e1b84723e22: !Template answer_choices: 1 ||| 2 - id: 82750934-e6fa-46b5-8d43-2cc53b3bcc9a + id: 8867366b-9956-4d75-a3fc-1e1b84723e22 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer 1 or 2. @@ -181,30 +150,29 @@ templates: original_task: true name: grammatical_which_one_1_2 reference: '' - af19a07b-836c-4a61-a195-91a3d3d3eefb: !Template + 8936931d-af83-498b-b4e6-1a1551f03206: !Template answer_choices: A ||| B - id: af19a07b-836c-4a61-a195-91a3d3d3eefb + id: 8936931d-af83-498b-b4e6-1a1551f03206 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -218,26 +186,26 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: better_B_A reference: '' - d5f377f8-8581-4858-95e5-890368ce7384: !Template - answer_choices: 1 ||| 2 - id: d5f377f8-8581-4858-95e5-890368ce7384 + 9b4dc3b2-723e-4b84-a8f9-f6e3794c9f0c: !Template + answer_choices: A ||| B + id: 9b4dc3b2-723e-4b84-a8f9-f6e3794c9f0c jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -257,24 +225,56 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: grammatical_which_one_A_B + reference: '' + b20d7f8e-e71d-4b26-9c56-35e121385e48: !Template + answer_choices: Yes ||| No + id: b20d7f8e-e71d-4b26-9c56-35e121385e48 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes + reference: '' + ee3b718f-6016-4488-b2be-7de1f775d3cb: !Template + answer_choices: Yes ||| No + id: ee3b718f-6016-4488-b2be-7de1f775d3cb + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no reference: '' - f27c2d94-0531-4717-9a66-6b59d8f06b91: !Template + f3542e20-350e-498a-b331-270b8dae4c25: !Template answer_choices: A ||| B - id: f27c2d94-0531-4717-9a66-6b59d8f06b91 + id: f3542e20-350e-498a-b331-270b8dae4c25 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + Sentence A: {{ sentence_good }} Sentence B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} Sentence B: {{ sentence_good }} diff --git a/promptsource/templates/blimp/animate_subject_trans/templates.yaml b/promptsource/templates/blimp/animate_subject_trans/templates.yaml index 4162644cc..6c189a8d6 100644 --- a/promptsource/templates/blimp/animate_subject_trans/templates.yaml +++ b/promptsource/templates/blimp/animate_subject_trans/templates.yaml @@ -1,30 +1,45 @@ dataset: blimp subset: animate_subject_trans templates: - 23cb5415-e6bb-422f-89d5-65d673b56b66: !Template + 32855303-3299-4dc0-98ba-db9c954c8deb: !Template + answer_choices: Yes ||| No + id: 32855303-3299-4dc0-98ba-db9c954c8deb + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 48f85408-43aa-4051-9811-7b88a59da2b1: !Template answer_choices: A ||| B - id: 23cb5415-e6bb-422f-89d5-65d673b56b66 + id: 48f85408-43aa-4051-9811-7b88a59da2b1 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -38,26 +53,26 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: better_B_A reference: '' - 41b54450-d6d8-44fe-8c9f-d9d75583f7e7: !Template - answer_choices: 1 ||| 2 - id: 41b54450-d6d8-44fe-8c9f-d9d75583f7e7 + 58930bb5-668c-4a7f-b54f-c0bf97cb1eb6: !Template + answer_choices: A ||| B + id: 58930bb5-668c-4a7f-b54f-c0bf97cb1eb6 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -77,47 +92,64 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: grammatical_between_A_B reference: '' - 53ed147d-3bbc-4887-8dc2-d146237098ed: !Template + 664030cf-5b46-48a2-afbb-83e6301574bf: !Template answer_choices: Yes ||| No - id: 53ed147d-3bbc-4887-8dc2-d146237098ed + id: 664030cf-5b46-48a2-afbb-83e6301574bf jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_bad_yes_no + name: single_sentence_good_no_yes + reference: '' + 6a607ce0-231e-4634-9430-7a734bf8c3a7: !Template + answer_choices: Yes ||| No + id: 6a607ce0-231e-4634-9430-7a734bf8c3a7 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no reference: '' - 750906cd-1c40-4e75-8715-8a27bacd10ac: !Template + b20918e6-936e-4fd2-afc2-c0a2324431ed: !Template answer_choices: A ||| B - id: 750906cd-1c40-4e75-8715-8a27bacd10ac + id: b20918e6-936e-4fd2-afc2-c0a2324431ed jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -131,32 +163,31 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: grammatical_which_one_A_B reference: '' - 87ed0b72-98c5-406d-8142-f93dc73268fd: !Template + ca52ffb8-898f-48d7-a9eb-8fc015fcf4e9: !Template answer_choices: A ||| B - id: 87ed0b72-98c5-406d-8142-f93dc73268fd + id: ca52ffb8-898f-48d7-a9eb-8fc015fcf4e9 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -170,31 +201,32 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: better_A_B reference: '' - 8e35221d-80fb-41c6-a730-18584e445119: !Template - answer_choices: A ||| B - id: 8e35221d-80fb-41c6-a730-18584e445119 + cada8bfd-4457-4e59-94a2-1c30878943fe: !Template + answer_choices: 1 ||| 2 + id: cada8bfd-4457-4e59-94a2-1c30878943fe jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -208,27 +240,11 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B - reference: '' - 97214438-78c8-45f7-b70c-2d13814d232f: !Template - answer_choices: Yes ||| No - id: 97214438-78c8-45f7-b70c-2d13814d232f - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes + name: grammatical_which_one_1_2 reference: '' - a5a3b920-9d1f-4813-b48f-26304d00725d: !Template + d853c50f-16e5-4cc0-9af6-13bbf6264a27: !Template answer_choices: Yes ||| No - id: a5a3b920-9d1f-4813-b48f-26304d00725d + id: d853c50f-16e5-4cc0-9af6-13bbf6264a27 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -242,9 +258,9 @@ templates: original_task: false name: single_sentence_bad_no_yes reference: '' - cc199813-7feb-4989-a35b-397ce431524e: !Template + e559f4b3-d385-42e4-8350-5f154c918242: !Template answer_choices: 1 ||| 2 - id: cc199813-7feb-4989-a35b-397ce431524e + id: e559f4b3-d385-42e4-8350-5f154c918242 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -281,19 +297,3 @@ templates: original_task: true name: grammatical_between_1_2 reference: '' - ddfc92aa-0952-4bfc-9631-cd0dd81fc85b: !Template - answer_choices: Yes ||| No - id: ddfc92aa-0952-4bfc-9631-cd0dd81fc85b - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no - reference: '' diff --git a/promptsource/templates/blimp/causative/templates.yaml b/promptsource/templates/blimp/causative/templates.yaml index 000728461..945285318 100644 --- a/promptsource/templates/blimp/causative/templates.yaml +++ b/promptsource/templates/blimp/causative/templates.yaml @@ -1,40 +1,24 @@ dataset: blimp subset: causative templates: - 15261315-cf12-4730-9cd1-29b065c485d8: !Template - answer_choices: Yes ||| No - id: 15261315-cf12-4730-9cd1-29b065c485d8 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes - reference: '' - 28913271-0dfa-4172-a2b2-81cc02a55740: !Template - answer_choices: A ||| B - id: 28913271-0dfa-4172-a2b2-81cc02a55740 + 012dd349-6fcc-4000-8f3b-3c15875a370e: !Template + answer_choices: 1 ||| 2 + id: 012dd349-6fcc-4000-8f3b-3c15875a370e jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -54,15 +38,15 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: grammatical_which_one_1_2 reference: '' - 311b5f2e-2595-4c2c-9238-d3a9b92ce09a: !Template + 09bce0c9-f733-41fe-b15d-c6984aef3e1f: !Template answer_choices: Yes ||| No - id: 311b5f2e-2595-4c2c-9238-d3a9b92ce09a + id: 09bce0c9-f733-41fe-b15d-c6984aef3e1f jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata @@ -70,30 +54,30 @@ templates: metrics: - Accuracy original_task: false - name: single_sentence_bad_yes_no + name: single_sentence_bad_no_yes reference: '' - 42efe756-1e77-4d04-8fbc-4a805cd25041: !Template + 232187ef-b6c2-44f2-affd-1aa152265d05: !Template answer_choices: A ||| B - id: 42efe756-1e77-4d04-8fbc-4a805cd25041 + id: 232187ef-b6c2-44f2-affd-1aa152265d05 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + Sentence A: {{ sentence_good }} Sentence B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} Sentence B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order + Which one is a better sentence of English, A or B? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -108,46 +92,30 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A - reference: '' - 4c5cf40e-e28c-47bb-a211-66ad8a2c5a25: !Template - answer_choices: Yes ||| No - id: 4c5cf40e-e28c-47bb-a211-66ad8a2c5a25 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no + name: better_A_B reference: '' - 5f94bead-d030-4bd3-b3ad-c609cfa1ecb6: !Template + 4479276c-a0ec-4b7a-bb9e-a1bc23a9bd3a: !Template answer_choices: A ||| B - id: 5f94bead-d030-4bd3-b3ad-c609cfa1ecb6 + id: 4479276c-a0ec-4b7a-bb9e-a1bc23a9bd3a jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + Sentence A: {{ sentence_good }} Sentence B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} Sentence B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order + Which one is a better sentence of English, B or A? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -162,11 +130,11 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: better_B_A reference: '' - b95c004b-a7bb-4abe-99b6-f1ceeff1997c: !Template + 78349830-65f8-4ec3-99c5-9b6c01b38358: !Template answer_choices: 1 ||| 2 - id: b95c004b-a7bb-4abe-99b6-f1ceeff1997c + id: 78349830-65f8-4ec3-99c5-9b6c01b38358 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -203,28 +171,12 @@ templates: original_task: true name: grammatical_between_1_2 reference: '' - cc192df1-d684-49ac-a4cc-ac53db84ae73: !Template - answer_choices: Yes ||| No - id: cc192df1-d684-49ac-a4cc-ac53db84ae73 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes - reference: '' - d19f516c-a99b-4388-8b5d-84731f619cee: !Template + 8fa1c4e1-1e82-47ad-abe7-5dc4c0d7c2ce: !Template answer_choices: A ||| B - id: d19f516c-a99b-4388-8b5d-84731f619cee + id: 8fa1c4e1-1e82-47ad-abe7-5dc4c0d7c2ce jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -256,26 +208,26 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: grammatical_between_A_B reference: '' - db00362c-7411-476d-9fe7-38a647a9a7e8: !Template - answer_choices: 1 ||| 2 - id: db00362c-7411-476d-9fe7-38a647a9a7e8 + 9c7358b2-4a6b-4575-a4a7-74f6396e4ff2: !Template + answer_choices: A ||| B + id: 9c7358b2-4a6b-4575-a4a7-74f6396e4ff2 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -295,5 +247,53 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: grammatical_which_one_A_B + reference: '' + b0fdd869-801e-4455-aeb7-19b145c11833: !Template + answer_choices: Yes ||| No + id: b0fdd869-801e-4455-aeb7-19b145c11833 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes + reference: '' + d9a76e7b-438a-4780-8045-50db5b00e9fb: !Template + answer_choices: Yes ||| No + id: d9a76e7b-438a-4780-8045-50db5b00e9fb + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + ec52732e-092c-48c9-8b6d-99e205f5ab8e: !Template + answer_choices: Yes ||| No + id: ec52732e-092c-48c9-8b6d-99e205f5ab8e + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no reference: '' diff --git a/promptsource/templates/blimp/complex_NP_island/templates.yaml b/promptsource/templates/blimp/complex_NP_island/templates.yaml index aa14a9b30..6482173d0 100644 --- a/promptsource/templates/blimp/complex_NP_island/templates.yaml +++ b/promptsource/templates/blimp/complex_NP_island/templates.yaml @@ -1,28 +1,60 @@ dataset: blimp subset: complex_NP_island templates: - 0495b672-0eeb-4486-8121-935a3bdb4a2d: !Template + 1de5ac17-e081-4c30-af42-be78b46550c5: !Template + answer_choices: Yes ||| No + id: 1de5ac17-e081-4c30-af42-be78b46550c5 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes + reference: '' + 49c874fd-6627-4788-aa79-e7416f3c8fbf: !Template + answer_choices: Yes ||| No + id: 49c874fd-6627-4788-aa79-e7416f3c8fbf + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 740e97d4-82c2-4502-a783-97639755143d: !Template answer_choices: A ||| B - id: 0495b672-0eeb-4486-8121-935a3bdb4a2d + id: 740e97d4-82c2-4502-a783-97639755143d jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + Sentence A: {{ sentence_good }} Sentence B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} Sentence B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order + Which one is a better sentence of English, B or A? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -37,26 +69,26 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: better_B_A reference: '' - 0dbb4a55-45cd-46b7-8a0a-78580888d312: !Template - answer_choices: A ||| B - id: 0dbb4a55-45cd-46b7-8a0a-78580888d312 + 950ea90c-c8d6-485a-916f-4f684c34ea89: !Template + answer_choices: 1 ||| 2 + id: 950ea90c-c8d6-485a-916f-4f684c34ea89 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -76,48 +108,31 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B - reference: '' - 0fb945e8-2f72-4622-a364-0716fe4c6a17: !Template - answer_choices: Yes ||| No - id: 0fb945e8-2f72-4622-a364-0716fe4c6a17 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes + name: grammatical_between_1_2 reference: '' - 296e0a9a-9eff-4ee9-9740-a22065d2b097: !Template - answer_choices: 1 ||| 2 - id: 296e0a9a-9eff-4ee9-9740-a22065d2b097 + a3fd445c-90fd-462b-9965-7a32b069b792: !Template + answer_choices: A ||| B + id: a3fd445c-90fd-462b-9965-7a32b069b792 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -131,47 +146,32 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 - reference: '' - 730ab874-f5d2-4527-9a7b-41f9474f6de1: !Template - answer_choices: Yes ||| No - id: 730ab874-f5d2-4527-9a7b-41f9474f6de1 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no + name: better_A_B reference: '' - 7beba882-e2e7-4f13-9599-f156d83b819c: !Template + c5f50d0a-e669-4061-85c9-d39e6e654d3c: !Template answer_choices: A ||| B - id: 7beba882-e2e7-4f13-9599-f156d83b819c + id: c5f50d0a-e669-4061-85c9-d39e6e654d3c jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -185,14 +185,30 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: grammatical_which_one_A_B reference: '' - 8798f8e4-7b90-435e-9630-fbfdb187ee46: !Template + cb4b4bec-0925-445e-b0d0-cc100a67016b: !Template + answer_choices: Yes ||| No + id: cb4b4bec-0925-445e-b0d0-cc100a67016b + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes + reference: '' + cdc0ba7d-d40b-4312-91e3-56638b2c3ba4: !Template answer_choices: A ||| B - id: 8798f8e4-7b90-435e-9630-fbfdb187ee46 + id: cdc0ba7d-d40b-4312-91e3-56638b2c3ba4 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -224,11 +240,11 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: grammatical_between_A_B reference: '' - b77195ba-a7e0-4225-a343-6f0239621f04: !Template + d082b242-cbb5-4e23-848c-479b34275ac3: !Template answer_choices: Yes ||| No - id: b77195ba-a7e0-4225-a343-6f0239621f04 + id: d082b242-cbb5-4e23-848c-479b34275ac3 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -242,25 +258,9 @@ templates: original_task: false name: single_sentence_bad_yes_no reference: '' - f09f244c-048a-4591-aee2-bff934b44cdd: !Template - answer_choices: Yes ||| No - id: f09f244c-048a-4591-aee2-bff934b44cdd - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes - reference: '' - f26a0feb-2759-4166-bb10-67337ca716a0: !Template + ed420a31-5808-4a93-8e1a-5b06e953973c: !Template answer_choices: 1 ||| 2 - id: f26a0feb-2759-4166-bb10-67337ca716a0 + id: ed420a31-5808-4a93-8e1a-5b06e953973c jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer 1 or 2. diff --git a/promptsource/templates/blimp/coordinate_structure_constraint_complex_left_branch/templates.yaml b/promptsource/templates/blimp/coordinate_structure_constraint_complex_left_branch/templates.yaml index 458a9aeee..2469f95f1 100644 --- a/promptsource/templates/blimp/coordinate_structure_constraint_complex_left_branch/templates.yaml +++ b/promptsource/templates/blimp/coordinate_structure_constraint_complex_left_branch/templates.yaml @@ -1,9 +1,25 @@ dataset: blimp subset: coordinate_structure_constraint_complex_left_branch templates: - 11622e6e-0c8c-49e7-b443-df3eb22aba58: !Template + 24075d3e-4262-4582-8b8a-0a4e7e74178c: !Template answer_choices: Yes ||| No - id: 11622e6e-0c8c-49e7-b443-df3eb22aba58 + id: 24075d3e-4262-4582-8b8a-0a4e7e74178c + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 24496016-429e-444a-8b01-9ec42e37af02: !Template + answer_choices: Yes ||| No + id: 24496016-429e-444a-8b01-9ec42e37af02 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -17,24 +33,24 @@ templates: original_task: false name: single_sentence_good_no_yes reference: '' - 30800a25-611a-48e3-b7cd-61484055d1da: !Template - answer_choices: 1 ||| 2 - id: 30800a25-611a-48e3-b7cd-61484055d1da + 412f33ac-7425-421c-aeef-cb3701bca420: !Template + answer_choices: A ||| B + id: 412f33ac-7425-421c-aeef-cb3701bca420 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -54,32 +70,31 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: grammatical_between_A_B reference: '' - 4d99c065-4b45-4ca2-9899-6630ec7c085d: !Template + 709eb4f6-8a40-4012-99f0-3054dde7b11b: !Template answer_choices: A ||| B - id: 4d99c065-4b45-4ca2-9899-6630ec7c085d + id: 709eb4f6-8a40-4012-99f0-3054dde7b11b jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -93,42 +108,26 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B - reference: '' - 73be937e-8c25-44f5-8acc-6db8329c7261: !Template - answer_choices: Yes ||| No - id: 73be937e-8c25-44f5-8acc-6db8329c7261 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no + name: better_A_B reference: '' - 7569112f-12f1-4e92-b01c-f24c9e2bf7f8: !Template - answer_choices: 1 ||| 2 - id: 7569112f-12f1-4e92-b01c-f24c9e2bf7f8 + 960c314c-f5ee-4350-b301-8a288b335458: !Template + answer_choices: A ||| B + id: 960c314c-f5ee-4350-b301-8a288b335458 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -148,11 +147,11 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: grammatical_which_one_A_B reference: '' - 84f173bc-8484-4070-9f38-ca3e837c4701: !Template + c1fd0ab7-e80b-4149-b198-5e4d07cf014f: !Template answer_choices: Yes ||| No - id: 84f173bc-8484-4070-9f38-ca3e837c4701 + id: c1fd0ab7-e80b-4149-b198-5e4d07cf014f jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -166,28 +165,28 @@ templates: original_task: false name: single_sentence_bad_no_yes reference: '' - b707cddf-b388-4b00-80ec-b99b401e00c3: !Template + c68f61f6-b53b-457c-99a3-5802157a9194: !Template answer_choices: A ||| B - id: b707cddf-b388-4b00-80ec-b99b401e00c3 + id: c68f61f6-b53b-457c-99a3-5802157a9194 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + Sentence A: {{ sentence_good }} Sentence B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} Sentence B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order + Which one is a better sentence of English, B or A? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -202,31 +201,32 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: better_B_A reference: '' - b86b71d6-2b94-4129-8b6a-254b4c1fe446: !Template - answer_choices: A ||| B - id: b86b71d6-2b94-4129-8b6a-254b4c1fe446 + c7fa415e-9375-44e4-9523-e81fe999ed7d: !Template + answer_choices: 1 ||| 2 + id: c7fa415e-9375-44e4-9523-e81fe999ed7d jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -240,26 +240,42 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: grammatical_between_1_2 reference: '' - cc916b85-8337-446a-87df-64a2b3917783: !Template - answer_choices: A ||| B - id: cc916b85-8337-446a-87df-64a2b3917783 + e2a36515-933c-435f-ace9-f664b0654717: !Template + answer_choices: Yes ||| No + id: e2a36515-933c-435f-ace9-f664b0654717 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + eae2f6d9-6105-4d5f-900c-d4faf2a68474: !Template + answer_choices: 1 ||| 2 + id: eae2f6d9-6105-4d5f-900c-d4faf2a68474 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -279,21 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B - reference: '' - db2bcadb-fcfe-4f86-946f-aa7d1f852d2e: !Template - answer_choices: Yes ||| No - id: db2bcadb-fcfe-4f86-946f-aa7d1f852d2e - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no + name: grammatical_which_one_1_2 reference: '' diff --git a/promptsource/templates/blimp/coordinate_structure_constraint_object_extraction/templates.yaml b/promptsource/templates/blimp/coordinate_structure_constraint_object_extraction/templates.yaml index fc33800d4..c60e8df62 100644 --- a/promptsource/templates/blimp/coordinate_structure_constraint_object_extraction/templates.yaml +++ b/promptsource/templates/blimp/coordinate_structure_constraint_object_extraction/templates.yaml @@ -1,9 +1,47 @@ dataset: blimp subset: coordinate_structure_constraint_object_extraction templates: - 1c49f77a-fe58-4439-9d94-0f4e1d3b6bef: !Template + 027b55c3-2903-4613-a898-33185f9f51b0: !Template + answer_choices: A ||| B + id: 027b55c3-2903-4613-a898-33185f9f51b0 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: better_A_B + reference: '' + 0297743c-253c-4b9c-bb56-d5248aa64fe9: !Template answer_choices: Yes ||| No - id: 1c49f77a-fe58-4439-9d94-0f4e1d3b6bef + id: 0297743c-253c-4b9c-bb56-d5248aa64fe9 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -17,22 +55,22 @@ templates: original_task: false name: single_sentence_good_yes_no reference: '' - 2fd5b072-efa8-4704-a830-d0af484a7bad: !Template + 0d28a4cf-03eb-4ce1-af7a-ff90e63d13bb: !Template answer_choices: A ||| B - id: 2fd5b072-efa8-4704-a830-d0af484a7bad + id: 0d28a4cf-03eb-4ce1-af7a-ff90e63d13bb jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + Sentence A: {{ sentence_good }} Sentence B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} Sentence B: {{ sentence_good }} @@ -55,24 +93,24 @@ templates: original_task: true name: better_B_A reference: '' - 32b4fcc3-22a2-4721-9c76-f4359f926274: !Template - answer_choices: A ||| B - id: 32b4fcc3-22a2-4721-9c76-f4359f926274 + 5b2491ae-f896-41f6-9712-6e44f4f4f8ac: !Template + answer_choices: 1 ||| 2 + id: 5b2491ae-f896-41f6-9712-6e44f4f4f8ac jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -92,15 +130,15 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: grammatical_between_1_2 reference: '' - 35813dd3-145c-4d35-800d-1f84f0a5b13d: !Template + 5f3ef743-71af-4dd4-8f50-abc340a09a48: !Template answer_choices: Yes ||| No - id: 35813dd3-145c-4d35-800d-1f84f0a5b13d + id: 5f3ef743-71af-4dd4-8f50-abc340a09a48 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata @@ -108,52 +146,30 @@ templates: metrics: - Accuracy original_task: false - name: single_sentence_bad_yes_no + name: single_sentence_bad_no_yes reference: '' - 87e44d3c-fff8-4dae-bb46-117f28266aa1: !Template - answer_choices: A ||| B - id: 87e44d3c-fff8-4dae-bb46-117f28266aa1 - jinja: '{% set shuffled_order = [0, 1] | random %} - - Read the following two sentences. - - {% if shuffled_order == 0 %} - - Sentence A: {{ sentence_good }} - - Sentence B: {{ sentence_bad }} - - {% else %} - - Sentence A: {{ sentence_bad }} - - Sentence B: {{ sentence_good }} - - {% endif %} - - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} + 6bc44dd4-2150-46f0-abb0-424d00679fd4: !Template + answer_choices: Yes ||| No + id: 6bc44dd4-2150-46f0-abb0-424d00679fd4 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - {% endif %}' + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: better_A_B + original_task: false + name: single_sentence_bad_yes_no reference: '' - 8a72ad22-1fb5-4d3d-af85-cfe0d676c8d8: !Template + 7f56b76f-9615-47a1-83d3-9c19e98481e5: !Template answer_choices: 1 ||| 2 - id: 8a72ad22-1fb5-4d3d-af85-cfe0d676c8d8 + id: 7f56b76f-9615-47a1-83d3-9c19e98481e5 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -185,14 +201,14 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: grammatical_which_one_1_2 reference: '' - 951278d2-eb6f-4c55-9ec1-ebd7d351c465: !Template + 92380e34-cddb-45ca-b8e6-a68185aa2728: !Template answer_choices: A ||| B - id: 951278d2-eb6f-4c55-9ec1-ebd7d351c465 + id: 92380e34-cddb-45ca-b8e6-a68185aa2728 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -224,26 +240,42 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: grammatical_between_A_B reference: '' - ea1b4ca5-c215-4f29-92bd-f60b169b6ff2: !Template - answer_choices: 1 ||| 2 - id: ea1b4ca5-c215-4f29-92bd-f60b169b6ff2 + a4d051ee-6bda-4def-80a8-021d696dee04: !Template + answer_choices: Yes ||| No + id: a4d051ee-6bda-4def-80a8-021d696dee04 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes + reference: '' + b32481b8-c3e1-48c1-9dd2-8582b730a8d8: !Template + answer_choices: A ||| B + id: b32481b8-c3e1-48c1-9dd2-8582b730a8d8 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -263,37 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 - reference: '' - f0817a39-597d-4fa5-8346-db551d9c5866: !Template - answer_choices: Yes ||| No - id: f0817a39-597d-4fa5-8346-db551d9c5866 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes - reference: '' - fcafa8ce-65ce-4b4d-8213-58c9f53558ec: !Template - answer_choices: Yes ||| No - id: fcafa8ce-65ce-4b4d-8213-58c9f53558ec - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes + name: grammatical_which_one_A_B reference: '' diff --git a/promptsource/templates/blimp/determiner_noun_agreement_1/templates.yaml b/promptsource/templates/blimp/determiner_noun_agreement_1/templates.yaml index 60a88d368..69733dcc2 100644 --- a/promptsource/templates/blimp/determiner_noun_agreement_1/templates.yaml +++ b/promptsource/templates/blimp/determiner_noun_agreement_1/templates.yaml @@ -1,45 +1,30 @@ dataset: blimp subset: determiner_noun_agreement_1 templates: - 01f50e2d-2102-4a35-a2cd-51d3b7592ddb: !Template - answer_choices: Yes ||| No - id: 01f50e2d-2102-4a35-a2cd-51d3b7592ddb - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no - reference: '' - 10ca8ed1-6c60-49c5-b8f3-2ae72288d97b: !Template - answer_choices: A ||| B - id: 10ca8ed1-6c60-49c5-b8f3-2ae72288d97b + 2c33dbe3-d023-4ed2-94f1-ad3100089999: !Template + answer_choices: 1 ||| 2 + id: 2c33dbe3-d023-4ed2-94f1-ad3100089999 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -53,14 +38,14 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: grammatical_which_one_1_2 reference: '' - 3729f629-d2a7-455e-8115-164873c05bfc: !Template + 33b939ff-3a68-4696-94b9-624c5bb99196: !Template answer_choices: 1 ||| 2 - id: 3729f629-d2a7-455e-8115-164873c05bfc + id: 33b939ff-3a68-4696-94b9-624c5bb99196 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -92,11 +77,11 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: grammatical_between_1_2 reference: '' - 45c96cc3-56bd-41f3-a986-f61991465b11: !Template + 534ca5a0-f7d7-44dd-bd5c-b560b8e527dc: !Template answer_choices: Yes ||| No - id: 45c96cc3-56bd-41f3-a986-f61991465b11 + id: 534ca5a0-f7d7-44dd-bd5c-b560b8e527dc jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -110,9 +95,9 @@ templates: original_task: false name: single_sentence_bad_yes_no reference: '' - 519b861a-03a9-4f99-a832-9d9992b34116: !Template + 82f50a6d-dfe8-4181-a17a-788051e83668: !Template answer_choices: Yes ||| No - id: 519b861a-03a9-4f99-a832-9d9992b34116 + id: 82f50a6d-dfe8-4181-a17a-788051e83668 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -126,12 +111,12 @@ templates: original_task: false name: single_sentence_bad_no_yes reference: '' - 8466dd95-2656-4061-9cfd-abe7d4e36771: !Template + 8bcb4d94-4645-4d17-a61f-ef01ce1d3224: !Template answer_choices: A ||| B - id: 8466dd95-2656-4061-9cfd-abe7d4e36771 + id: 8bcb4d94-4645-4d17-a61f-ef01ce1d3224 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -163,24 +148,40 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: grammatical_between_A_B + reference: '' + 90380c6c-6997-489e-ad66-294712be986f: !Template + answer_choices: Yes ||| No + id: 90380c6c-6997-489e-ad66-294712be986f + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes reference: '' - 9d7f5a05-b76f-4ebc-93f2-194e438eab02: !Template + 9d442c90-74a2-49e4-afb2-083652d89a95: !Template answer_choices: A ||| B - id: 9d7f5a05-b76f-4ebc-93f2-194e438eab02 + id: 9d442c90-74a2-49e4-afb2-083652d89a95 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + Sentence A: {{ sentence_good }} Sentence B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} Sentence B: {{ sentence_good }} @@ -203,30 +204,29 @@ templates: original_task: true name: better_A_B reference: '' - ba5f23a8-d37b-40cb-aa6b-2fcd72269b2f: !Template + a3170a13-b9eb-445f-bb4a-c202cd9abbec: !Template answer_choices: A ||| B - id: ba5f23a8-d37b-40cb-aa6b-2fcd72269b2f + id: a3170a13-b9eb-445f-bb4a-c202cd9abbec jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -240,15 +240,15 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: better_B_A reference: '' - d16ebcab-6ba3-4b8c-b3d4-d2065fa56b0f: !Template + a7b7f359-b815-4e92-87d8-ab8e3b07121d: !Template answer_choices: Yes ||| No - id: d16ebcab-6ba3-4b8c-b3d4-d2065fa56b0f + id: a7b7f359-b815-4e92-87d8-ab8e3b07121d jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -256,26 +256,26 @@ templates: metrics: - Accuracy original_task: false - name: single_sentence_good_no_yes + name: single_sentence_good_yes_no reference: '' - ed5abdcf-8f64-4073-9f3a-6b06ad93c053: !Template - answer_choices: 1 ||| 2 - id: ed5abdcf-8f64-4073-9f3a-6b06ad93c053 + f2023fa2-5bdc-45fa-934e-e517b41fcea5: !Template + answer_choices: A ||| B + id: f2023fa2-5bdc-45fa-934e-e517b41fcea5 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -295,5 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: grammatical_which_one_A_B reference: '' diff --git a/promptsource/templates/blimp/determiner_noun_agreement_2/templates.yaml b/promptsource/templates/blimp/determiner_noun_agreement_2/templates.yaml index 33f9bb024..ba1c6b248 100644 --- a/promptsource/templates/blimp/determiner_noun_agreement_2/templates.yaml +++ b/promptsource/templates/blimp/determiner_noun_agreement_2/templates.yaml @@ -1,24 +1,40 @@ dataset: blimp subset: determiner_noun_agreement_2 templates: - 0abbea35-91b3-479b-9aff-389b6cc2b9fe: !Template - answer_choices: 1 ||| 2 - id: 0abbea35-91b3-479b-9aff-389b6cc2b9fe + 287f554f-0296-4745-854f-55a7f0615c1a: !Template + answer_choices: Yes ||| No + id: 287f554f-0296-4745-854f-55a7f0615c1a + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 3ba5ee24-2973-463c-863a-d3d17ea1cbe8: !Template + answer_choices: A ||| B + id: 3ba5ee24-2973-463c-863a-d3d17ea1cbe8 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -38,63 +54,32 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 - reference: '' - 155402ce-a653-413d-a9bc-e4f1c8d6e4f7: !Template - answer_choices: Yes ||| No - id: 155402ce-a653-413d-a9bc-e4f1c8d6e4f7 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no - reference: '' - 247f5291-8873-43cb-b523-0ee00bb56a69: !Template - answer_choices: Yes ||| No - id: 247f5291-8873-43cb-b523-0ee00bb56a69 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes + name: grammatical_between_A_B reference: '' - 33d6830a-d214-4ff8-80fd-553bdbedccfa: !Template + 3d19ee7e-f9ce-4eee-8a54-f5175ff2d32b: !Template answer_choices: A ||| B - id: 33d6830a-d214-4ff8-80fd-553bdbedccfa + id: 3d19ee7e-f9ce-4eee-8a54-f5175ff2d32b jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -108,15 +93,15 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: grammatical_which_one_A_B reference: '' - 3ea31cb4-bc35-42fe-80f7-4125ba58b309: !Template + 5898404e-1736-4df5-bdc6-e9d40467138d: !Template answer_choices: Yes ||| No - id: 3ea31cb4-bc35-42fe-80f7-4125ba58b309 + id: 5898404e-1736-4df5-bdc6-e9d40467138d jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -124,48 +109,31 @@ templates: metrics: - Accuracy original_task: false - name: single_sentence_good_yes_no - reference: '' - 5547dba3-b482-427b-9fbe-8491bdca060a: !Template - answer_choices: Yes ||| No - id: 5547dba3-b482-427b-9fbe-8491bdca060a - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes + name: single_sentence_good_no_yes reference: '' - 60f34769-c923-419c-bd42-fc289201eace: !Template + 924bfa4b-e6bd-45fa-af65-8f94ab33ea78: !Template answer_choices: A ||| B - id: 60f34769-c923-419c-bd42-fc289201eace + id: 924bfa4b-e6bd-45fa-af65-8f94ab33ea78 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -179,14 +147,14 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: better_B_A reference: '' - 6ed8dd47-f3bf-4365-a0ad-2dde1bace547: !Template + aea1cc03-f411-4d5b-bba6-140c30fa6c46: !Template answer_choices: 1 ||| 2 - id: 6ed8dd47-f3bf-4365-a0ad-2dde1bace547 + id: aea1cc03-f411-4d5b-bba6-140c30fa6c46 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -218,26 +186,26 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: grammatical_between_1_2 reference: '' - 95573879-a945-41c4-9a02-e256879622b9: !Template - answer_choices: A ||| B - id: 95573879-a945-41c4-9a02-e256879622b9 + b64965c6-e672-4e48-adfb-95e75b9ce537: !Template + answer_choices: 1 ||| 2 + id: b64965c6-e672-4e48-adfb-95e75b9ce537 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -257,30 +225,30 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: grammatical_which_one_1_2 reference: '' - a610bc03-9da1-482e-93bc-948c5bbc560d: !Template + d2394fd4-954f-4b34-9b14-680a6b855fb2: !Template answer_choices: A ||| B - id: a610bc03-9da1-482e-93bc-948c5bbc560d + id: d2394fd4-954f-4b34-9b14-680a6b855fb2 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + Sentence A: {{ sentence_good }} Sentence B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} Sentence B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order + Which one is a better sentence of English, A or B? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -295,5 +263,37 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: better_A_B + reference: '' + ed64a5f2-73c8-4c8d-8fe8-f34023bec7dd: !Template + answer_choices: Yes ||| No + id: ed64a5f2-73c8-4c8d-8fe8-f34023bec7dd + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + ef8c6db8-d1e0-48ae-8065-3c140ea3bc0e: !Template + answer_choices: Yes ||| No + id: ef8c6db8-d1e0-48ae-8065-3c140ea3bc0e + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes reference: '' diff --git a/promptsource/templates/blimp/determiner_noun_agreement_irregular_1/templates.yaml b/promptsource/templates/blimp/determiner_noun_agreement_irregular_1/templates.yaml index add4dae6e..d33b925b1 100644 --- a/promptsource/templates/blimp/determiner_noun_agreement_irregular_1/templates.yaml +++ b/promptsource/templates/blimp/determiner_noun_agreement_irregular_1/templates.yaml @@ -1,22 +1,38 @@ dataset: blimp subset: determiner_noun_agreement_irregular_1 templates: - 0be7739d-5820-471d-ab43-14e5ea6baee9: !Template + 281b27db-76af-4a85-8f3d-a34d77a43b76: !Template + answer_choices: Yes ||| No + id: 281b27db-76af-4a85-8f3d-a34d77a43b76 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 331683dc-bdef-4a2a-8f13-b9da7eaab72b: !Template answer_choices: A ||| B - id: 0be7739d-5820-471d-ab43-14e5ea6baee9 + id: 331683dc-bdef-4a2a-8f13-b9da7eaab72b jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + Sentence A: {{ sentence_good }} Sentence B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} Sentence B: {{ sentence_good }} @@ -39,40 +55,24 @@ templates: original_task: true name: better_B_A reference: '' - 3df7c754-c0f9-403b-9dec-9de4b0d08b10: !Template - answer_choices: Yes ||| No - id: 3df7c754-c0f9-403b-9dec-9de4b0d08b10 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes - reference: '' - 4a936cf9-76f9-4905-9bb0-a2e501fc26d3: !Template - answer_choices: 1 ||| 2 - id: 4a936cf9-76f9-4905-9bb0-a2e501fc26d3 + 3b852bc6-1d1e-44bb-98a1-ec2675ab5eb2: !Template + answer_choices: A ||| B + id: 3b852bc6-1d1e-44bb-98a1-ec2675ab5eb2 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -92,24 +92,24 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: grammatical_which_one_A_B reference: '' - 639e489a-2d99-44d8-9d88-a652437ecda7: !Template + 5757fd44-5409-4fad-90c5-ccea3750ac7e: !Template answer_choices: A ||| B - id: 639e489a-2d99-44d8-9d88-a652437ecda7 + id: 5757fd44-5409-4fad-90c5-ccea3750ac7e jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + Sentence A: {{ sentence_good }} Sentence B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} Sentence B: {{ sentence_good }} @@ -132,12 +132,12 @@ templates: original_task: true name: better_A_B reference: '' - 7367f407-4707-4b75-91a7-6a49fa0faffc: !Template + 58a0dba8-1840-476d-a345-f8b182a13c00: !Template answer_choices: 1 ||| 2 - id: 7367f407-4707-4b75-91a7-6a49fa0faffc + id: 58a0dba8-1840-476d-a345-f8b182a13c00 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -169,11 +169,50 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: grammatical_which_one_1_2 + reference: '' + 843a4a4d-823d-4045-bdae-0e58e6817876: !Template + answer_choices: A ||| B + id: 843a4a4d-823d-4045-bdae-0e58e6817876 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_A_B reference: '' - aaff53b9-8a57-433d-a7a5-c4935a8ca799: !Template + 882264b2-255b-4796-bacd-8f9f1130cdc1: !Template answer_choices: Yes ||| No - id: aaff53b9-8a57-433d-a7a5-c4935a8ca799 + id: 882264b2-255b-4796-bacd-8f9f1130cdc1 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -187,9 +226,9 @@ templates: original_task: false name: single_sentence_bad_no_yes reference: '' - bf3f6215-b44d-42a2-a0ba-ab139d18bd06: !Template + 93ced979-9e52-46b5-b328-d3729b2a541a: !Template answer_choices: Yes ||| No - id: bf3f6215-b44d-42a2-a0ba-ab139d18bd06 + id: 93ced979-9e52-46b5-b328-d3729b2a541a jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -203,13 +242,13 @@ templates: original_task: false name: single_sentence_bad_yes_no reference: '' - c26344c9-cc1e-4820-9bb8-426583791567: !Template + ab3ae97a-002c-4f3b-a9f1-5a2e9413ed8a: !Template answer_choices: Yes ||| No - id: c26344c9-cc1e-4820-9bb8-426583791567 + id: ab3ae97a-002c-4f3b-a9f1-5a2e9413ed8a jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -217,65 +256,26 @@ templates: metrics: - Accuracy original_task: false - name: single_sentence_good_yes_no - reference: '' - c2959e6a-353a-4195-99e6-1dd49e1046ba: !Template - answer_choices: A ||| B - id: c2959e6a-353a-4195-99e6-1dd49e1046ba - jinja: '{% set shuffled_order = [0, 1] | random %} - - Which one of the following sentences is grammatical? Please answer A or B. - - {% if shuffled_order == 0 %} - - A: {{ sentence_good }} - - B: {{ sentence_bad }} - - {% else %} - - A: {{ sentence_bad }} - - B: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} - - {% endif %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: grammatical_which_one_A_B + name: single_sentence_good_no_yes reference: '' - e6c55627-465f-40f4-bae6-248afeced026: !Template - answer_choices: A ||| B - id: e6c55627-465f-40f4-bae6-248afeced026 + f065fb5a-2bda-4690-b52b-b6abed4c525d: !Template + answer_choices: 1 ||| 2 + id: f065fb5a-2bda-4690-b52b-b6abed4c525d jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -295,5 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: grammatical_between_1_2 reference: '' diff --git a/promptsource/templates/blimp/determiner_noun_agreement_irregular_2/templates.yaml b/promptsource/templates/blimp/determiner_noun_agreement_irregular_2/templates.yaml index 0506ba69e..3f4afcb02 100644 --- a/promptsource/templates/blimp/determiner_noun_agreement_irregular_2/templates.yaml +++ b/promptsource/templates/blimp/determiner_noun_agreement_irregular_2/templates.yaml @@ -1,9 +1,9 @@ dataset: blimp subset: determiner_noun_agreement_irregular_2 templates: - 26515490-1232-4b88-abd0-26e6e9d5fa56: !Template + 295e6c73-26c0-46f1-89ea-fb3f4555c659: !Template answer_choices: Yes ||| No - id: 26515490-1232-4b88-abd0-26e6e9d5fa56 + id: 295e6c73-26c0-46f1-89ea-fb3f4555c659 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -17,30 +17,29 @@ templates: original_task: false name: single_sentence_good_yes_no reference: '' - 4566b72e-cfa9-4cc9-ade5-3164d9740d75: !Template - answer_choices: 1 ||| 2 - id: 4566b72e-cfa9-4cc9-ade5-3164d9740d75 + 36ec8691-698f-4905-8cef-b848e3bd9cd5: !Template + answer_choices: A ||| B + id: 36ec8691-698f-4905-8cef-b848e3bd9cd5 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -54,26 +53,26 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: better_A_B reference: '' - 4f5e375d-1f8d-432e-b9f9-a2f0eda9cc3e: !Template - answer_choices: A ||| B - id: 4f5e375d-1f8d-432e-b9f9-a2f0eda9cc3e + 53d4645d-34b7-487c-b95d-ce88d37dd9fc: !Template + answer_choices: 1 ||| 2 + id: 53d4645d-34b7-487c-b95d-ce88d37dd9fc jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -93,31 +92,64 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: grammatical_which_one_1_2 reference: '' - 5043d1cb-4265-4cec-9510-d95f0dca1a31: !Template + 5ead7bd5-831c-48aa-871f-1d7c13b6548e: !Template + answer_choices: Yes ||| No + id: 5ead7bd5-831c-48aa-871f-1d7c13b6548e + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 6341fd3c-2768-40d2-b9fb-6f69d2baf250: !Template + answer_choices: Yes ||| No + id: 6341fd3c-2768-40d2-b9fb-6f69d2baf250 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes + reference: '' + 6fd90f4e-72cb-4cc4-8d3c-4c18b8a30910: !Template answer_choices: A ||| B - id: 5043d1cb-4265-4cec-9510-d95f0dca1a31 + id: 6fd90f4e-72cb-4cc4-8d3c-4c18b8a30910 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -131,32 +163,31 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: grammatical_between_A_B reference: '' - 60b2cf66-c88b-4e08-9f9d-6f25134e0be3: !Template + 92ba313f-cefd-4a6f-8449-314bd4dfcbbe: !Template answer_choices: A ||| B - id: 60b2cf66-c88b-4e08-9f9d-6f25134e0be3 + id: 92ba313f-cefd-4a6f-8449-314bd4dfcbbe jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -170,42 +201,42 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: better_B_A reference: '' - 6f982cc1-127e-4ba0-b757-4c7f0d3b1c48: !Template + 935f5b0c-35bb-41e7-8b7f-f7a632963779: !Template answer_choices: Yes ||| No - id: 6f982cc1-127e-4ba0-b757-4c7f0d3b1c48 + id: 935f5b0c-35bb-41e7-8b7f-f7a632963779 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_bad_yes_no + name: single_sentence_good_no_yes reference: '' - 7d2de496-10ea-47fb-b38f-44a6856d3764: !Template - answer_choices: 1 ||| 2 - id: 7d2de496-10ea-47fb-b38f-44a6856d3764 + c1413ff7-c645-4009-8c9f-f8d62e1c6ca6: !Template + answer_choices: A ||| B + id: c1413ff7-c645-4009-8c9f-f8d62e1c6ca6 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -225,47 +256,32 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 - reference: '' - ab9d74f5-4504-41bc-b3b4-cf2c624f1fe2: !Template - answer_choices: Yes ||| No - id: ab9d74f5-4504-41bc-b3b4-cf2c624f1fe2 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes + name: grammatical_which_one_A_B reference: '' - b05688a5-ca11-4bcd-af6f-b9d21ff14294: !Template - answer_choices: A ||| B - id: b05688a5-ca11-4bcd-af6f-b9d21ff14294 + c9de7a9e-9c14-4071-a5f3-8899f7fe5702: !Template + answer_choices: 1 ||| 2 + id: c9de7a9e-9c14-4071-a5f3-8899f7fe5702 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -279,21 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A - reference: '' - f990337c-829d-43d1-91a1-87e9bff30b7a: !Template - answer_choices: Yes ||| No - id: f990337c-829d-43d1-91a1-87e9bff30b7a - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes + name: grammatical_between_1_2 reference: '' diff --git a/promptsource/templates/blimp/determiner_noun_agreement_with_adj_2/templates.yaml b/promptsource/templates/blimp/determiner_noun_agreement_with_adj_2/templates.yaml index 8a69962c7..1ce1d6f2c 100644 --- a/promptsource/templates/blimp/determiner_noun_agreement_with_adj_2/templates.yaml +++ b/promptsource/templates/blimp/determiner_noun_agreement_with_adj_2/templates.yaml @@ -1,28 +1,28 @@ dataset: blimp subset: determiner_noun_agreement_with_adj_2 templates: - 2702ea88-ffe0-4176-b033-1fb6b022b0f2: !Template + 1cc44f9b-dc96-4231-8a7c-c57ff1a124dc: !Template answer_choices: Yes ||| No - id: 2702ea88-ffe0-4176-b033-1fb6b022b0f2 + id: 1cc44f9b-dc96-4231-8a7c-c57ff1a124dc jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_good_yes_no + name: single_sentence_bad_yes_no reference: '' - 4306aa5e-6363-4853-aa80-2504a802930c: !Template + 3b2b6cfb-5b6a-41a8-9aec-35fdca3044e9: !Template answer_choices: 1 ||| 2 - id: 4306aa5e-6363-4853-aa80-2504a802930c + id: 3b2b6cfb-5b6a-41a8-9aec-35fdca3044e9 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -54,26 +54,26 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: grammatical_between_1_2 reference: '' - 45eba5a5-d68d-4587-a641-ff398dc9005f: !Template - answer_choices: A ||| B - id: 45eba5a5-d68d-4587-a641-ff398dc9005f + 8e9baedd-f5cd-4a8c-b66e-3f40149f38cf: !Template + answer_choices: 1 ||| 2 + id: 8e9baedd-f5cd-4a8c-b66e-3f40149f38cf jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -93,11 +93,43 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: grammatical_which_one_1_2 reference: '' - 64c92ead-51cd-43da-94bb-54000b7e8798: !Template + 8f282b1d-077e-4dc6-9f42-0a466bf43a87: !Template + answer_choices: Yes ||| No + id: 8f282b1d-077e-4dc6-9f42-0a466bf43a87 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes + reference: '' + a08cff29-52d7-4897-804a-39fc6743a1a2: !Template + answer_choices: Yes ||| No + id: a08cff29-52d7-4897-804a-39fc6743a1a2 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes + reference: '' + be3b9b9d-b82c-4450-87a8-3fbd0aa1230c: !Template answer_choices: A ||| B - id: 64c92ead-51cd-43da-94bb-54000b7e8798 + id: be3b9b9d-b82c-4450-87a8-3fbd0aa1230c jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer A or B. @@ -134,22 +166,22 @@ templates: original_task: true name: grammatical_between_A_B reference: '' - 73d3e906-e9af-4f24-a9cb-128dd81b7dd9: !Template + c5422144-4e75-4c66-84ca-57fcc4233a5b: !Template answer_choices: A ||| B - id: 73d3e906-e9af-4f24-a9cb-128dd81b7dd9 + id: c5422144-4e75-4c66-84ca-57fcc4233a5b jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + Sentence A: {{ sentence_good }} Sentence B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} Sentence B: {{ sentence_good }} @@ -172,62 +204,29 @@ templates: original_task: true name: better_B_A reference: '' - 89179eb8-24f0-44f7-a603-dc44db267b9c: !Template - answer_choices: Yes ||| No - id: 89179eb8-24f0-44f7-a603-dc44db267b9c - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes - reference: '' - 89dd1889-2341-42c8-9354-47aa9747da7b: !Template - answer_choices: Yes ||| No - id: 89dd1889-2341-42c8-9354-47aa9747da7b - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes - reference: '' - d5dc95bc-088d-46e7-a22f-e34f038af42e: !Template - answer_choices: 1 ||| 2 - id: d5dc95bc-088d-46e7-a22f-e34f038af42e + ca8ff566-c1fa-4ae1-acfc-5110cdf477eb: !Template + answer_choices: A ||| B + id: ca8ff566-c1fa-4ae1-acfc-5110cdf477eb jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -241,31 +240,32 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: better_A_B reference: '' - dcbe84c8-fe8f-450f-8f70-4bf34d21b6a4: !Template + d4eb6764-77bf-4e3c-9024-d276dfd7880f: !Template answer_choices: A ||| B - id: dcbe84c8-fe8f-450f-8f70-4bf34d21b6a4 + id: d4eb6764-77bf-4e3c-9024-d276dfd7880f jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -279,21 +279,21 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: grammatical_which_one_A_B reference: '' - ea091a0b-0ec2-4a44-aaeb-3a6d1772e477: !Template + d7545db3-84b6-4d0e-93e0-9d77905ac656: !Template answer_choices: Yes ||| No - id: ea091a0b-0ec2-4a44-aaeb-3a6d1772e477 + id: d7545db3-84b6-4d0e-93e0-9d77905ac656 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_bad_yes_no + name: single_sentence_good_yes_no reference: '' diff --git a/promptsource/templates/blimp/determiner_noun_agreement_with_adj_irregular_1/templates.yaml b/promptsource/templates/blimp/determiner_noun_agreement_with_adj_irregular_1/templates.yaml index 66f7a647b..a32f5b483 100644 --- a/promptsource/templates/blimp/determiner_noun_agreement_with_adj_irregular_1/templates.yaml +++ b/promptsource/templates/blimp/determiner_noun_agreement_with_adj_irregular_1/templates.yaml @@ -1,22 +1,22 @@ dataset: blimp subset: determiner_noun_agreement_with_adj_irregular_1 templates: - 1dce2f7c-1341-497b-b824-c24120fd66d7: !Template + 41a3a39d-5c4d-4988-a800-b03d740498bd: !Template answer_choices: A ||| B - id: 1dce2f7c-1341-497b-b824-c24120fd66d7 + id: 41a3a39d-5c4d-4988-a800-b03d740498bd jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + Sentence A: {{ sentence_good }} Sentence B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} Sentence B: {{ sentence_good }} @@ -39,61 +39,38 @@ templates: original_task: true name: better_A_B reference: '' - 2339eda9-fc03-4207-a71b-a5cc01b2e13a: !Template - answer_choices: 1 ||| 2 - id: 2339eda9-fc03-4207-a71b-a5cc01b2e13a - jinja: '{% set shuffled_order = [0, 1] | random %} - - Between the two sentences, which one is grammatical? Please answer 1 or 2. - - {% if shuffled_order == 0 %} - - 1: {{ sentence_good }} - - 2: {{ sentence_bad }} - - {% else %} - - 1: {{ sentence_bad }} - - 2: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} + 5f53637e-06a1-426c-992a-62f6fcb6cabc: !Template + answer_choices: Yes ||| No + id: 5f53637e-06a1-426c-992a-62f6fcb6cabc + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - {% endif %}' + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: grammatical_between_1_2 + original_task: false + name: single_sentence_good_no_yes reference: '' - 38c55b36-426a-4001-86dd-391c643adab5: !Template + 73662fd0-ec0e-40b3-bc9e-fa35fba20815: !Template answer_choices: A ||| B - id: 38c55b36-426a-4001-86dd-391c643adab5 + id: 73662fd0-ec0e-40b3-bc9e-fa35fba20815 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + Sentence A: {{ sentence_good }} Sentence B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} Sentence B: {{ sentence_good }} @@ -116,13 +93,13 @@ templates: original_task: true name: better_B_A reference: '' - 47e22a97-df75-41c9-b1f1-28cf9c99ea91: !Template + 7f0fda4d-a15d-4dc2-a516-31aa0e099ade: !Template answer_choices: Yes ||| No - id: 47e22a97-df75-41c9-b1f1-28cf9c99ea91 + id: 7f0fda4d-a15d-4dc2-a516-31aa0e099ade jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata @@ -130,11 +107,11 @@ templates: metrics: - Accuracy original_task: false - name: single_sentence_bad_yes_no + name: single_sentence_bad_no_yes reference: '' - 59f3abfa-7cf2-48a5-9c0c-bc5467e8c6d9: !Template + 981b7c2e-139e-448e-b0e7-d55b75497e37: !Template answer_choices: Yes ||| No - id: 59f3abfa-7cf2-48a5-9c0c-bc5467e8c6d9 + id: 981b7c2e-139e-448e-b0e7-d55b75497e37 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -148,24 +125,24 @@ templates: original_task: false name: single_sentence_good_yes_no reference: '' - 5ace59df-aa43-479b-b2eb-ea2b7d1204ba: !Template - answer_choices: A ||| B - id: 5ace59df-aa43-479b-b2eb-ea2b7d1204ba + a72f3304-f283-4449-8298-576e8446d1fd: !Template + answer_choices: 1 ||| 2 + id: a72f3304-f283-4449-8298-576e8446d1fd jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -185,30 +162,14 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B - reference: '' - 6d08e8d4-769f-4e1c-b436-f5663f0c9a54: !Template - answer_choices: Yes ||| No - id: 6d08e8d4-769f-4e1c-b436-f5663f0c9a54 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes + name: grammatical_which_one_1_2 reference: '' - b8e57aff-fbb4-4016-afba-048ec42f6ebf: !Template + cebaa4c0-dadb-4e94-8f6f-896ea8a1f1dc: !Template answer_choices: 1 ||| 2 - id: b8e57aff-fbb4-4016-afba-048ec42f6ebf + id: cebaa4c0-dadb-4e94-8f6f-896ea8a1f1dc jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -240,27 +201,50 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: grammatical_between_1_2 reference: '' - e541b31d-9263-4dff-86d1-d536f367c42f: !Template - answer_choices: Yes ||| No - id: e541b31d-9263-4dff-86d1-d536f367c42f - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + d2b12553-0384-4aee-b357-0fd004d7df11: !Template + answer_choices: A ||| B + id: d2b12553-0384-4aee-b357-0fd004d7df11 + jinja: '{% set shuffled_order = [0, 1] | random %} - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: false - name: single_sentence_good_no_yes + original_task: true + name: grammatical_between_A_B reference: '' - f8d2ddd5-b969-4086-9bf0-a7fa11020a13: !Template + f127b1d1-8bc9-46b3-ad43-ef83a6182721: !Template answer_choices: A ||| B - id: f8d2ddd5-b969-4086-9bf0-a7fa11020a13 + id: f127b1d1-8bc9-46b3-ad43-ef83a6182721 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer A or B. @@ -297,3 +281,19 @@ templates: original_task: true name: grammatical_which_one_A_B reference: '' + f17dccfc-874d-4073-b0da-934539891bca: !Template + answer_choices: Yes ||| No + id: f17dccfc-874d-4073-b0da-934539891bca + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' diff --git a/promptsource/templates/blimp/determiner_noun_agreement_with_adj_irregular_2/templates.yaml b/promptsource/templates/blimp/determiner_noun_agreement_with_adj_irregular_2/templates.yaml index 4fe0fdbbb..1e69784b5 100644 --- a/promptsource/templates/blimp/determiner_noun_agreement_with_adj_irregular_2/templates.yaml +++ b/promptsource/templates/blimp/determiner_noun_agreement_with_adj_irregular_2/templates.yaml @@ -1,24 +1,40 @@ dataset: blimp subset: determiner_noun_agreement_with_adj_irregular_2 templates: - 1e5ff764-2666-4a46-bdf7-bbc53e945186: !Template - answer_choices: 1 ||| 2 - id: 1e5ff764-2666-4a46-bdf7-bbc53e945186 + 2354b1e1-f868-4332-bce7-66041e126ae8: !Template + answer_choices: Yes ||| No + id: 2354b1e1-f868-4332-bce7-66041e126ae8 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 581fa102-823c-4a0e-a510-f51fbf2bdece: !Template + answer_choices: A ||| B + id: 581fa102-823c-4a0e-a510-f51fbf2bdece jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -38,31 +54,32 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: grammatical_between_A_B reference: '' - 44ef80e5-725b-40cf-b7b7-db754c332682: !Template + 5a66a33a-fd03-4b54-b367-24b7bb0ec423: !Template answer_choices: A ||| B - id: 44ef80e5-725b-40cf-b7b7-db754c332682 + id: 5a66a33a-fd03-4b54-b367-24b7bb0ec423 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -76,48 +93,31 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B - reference: '' - 53352573-715b-4152-a73c-0a9b58325c8f: !Template - answer_choices: Yes ||| No - id: 53352573-715b-4152-a73c-0a9b58325c8f - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes + name: grammatical_which_one_A_B reference: '' - 6afacf5a-d598-426f-8b82-bcb6a6ab5ee5: !Template + 6b583fbb-5034-4ce1-81c6-58d655661e22: !Template answer_choices: A ||| B - id: 6afacf5a-d598-426f-8b82-bcb6a6ab5ee5 + id: 6b583fbb-5034-4ce1-81c6-58d655661e22 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -131,15 +131,15 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: better_A_B reference: '' - 8b2223e4-1a76-4bb1-8856-5aeb8b0ae838: !Template + 75afa7ac-1293-4ac0-b8fb-a6ed466e4329: !Template answer_choices: Yes ||| No - id: 8b2223e4-1a76-4bb1-8856-5aeb8b0ae838 + id: 75afa7ac-1293-4ac0-b8fb-a6ed466e4329 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -147,11 +147,27 @@ templates: metrics: - Accuracy original_task: false - name: single_sentence_good_yes_no + name: single_sentence_good_no_yes reference: '' - 94b67102-49c3-41d1-9c85-08043921617c: !Template + 823a5fa9-8c51-4d59-958f-eedf890797a6: !Template + answer_choices: Yes ||| No + id: 823a5fa9-8c51-4d59-958f-eedf890797a6 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes + reference: '' + 97e75645-7f2b-4490-8959-06de683d02a6: !Template answer_choices: 1 ||| 2 - id: 94b67102-49c3-41d1-9c85-08043921617c + id: 97e75645-7f2b-4490-8959-06de683d02a6 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer 1 or 2. @@ -188,9 +204,9 @@ templates: original_task: true name: grammatical_which_one_1_2 reference: '' - a3562e15-369b-4fa2-919d-e8dfe5b99df6: !Template + ced385f4-6a92-4f90-813f-7cc2cc47ee4b: !Template answer_choices: Yes ||| No - id: a3562e15-369b-4fa2-919d-e8dfe5b99df6 + id: ced385f4-6a92-4f90-813f-7cc2cc47ee4b jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -204,22 +220,22 @@ templates: original_task: false name: single_sentence_bad_yes_no reference: '' - c11f15e2-1678-440d-a6ba-ed4630942049: !Template + e8f7caae-a9b3-434e-a033-0405636d26d7: !Template answer_choices: A ||| B - id: c11f15e2-1678-440d-a6ba-ed4630942049 + id: e8f7caae-a9b3-434e-a033-0405636d26d7 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + Sentence A: {{ sentence_good }} Sentence B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} Sentence B: {{ sentence_good }} @@ -242,40 +258,24 @@ templates: original_task: true name: better_B_A reference: '' - c1cd46ac-c799-4f28-a72c-e51d4a02a5b5: !Template - answer_choices: Yes ||| No - id: c1cd46ac-c799-4f28-a72c-e51d4a02a5b5 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes - reference: '' - e6215cbd-8dde-46b3-b79b-b869eb7f2c4b: !Template - answer_choices: A ||| B - id: e6215cbd-8dde-46b3-b79b-b869eb7f2c4b + fd8b6ded-6cac-452d-89fc-7e18fb70658d: !Template + answer_choices: 1 ||| 2 + id: fd8b6ded-6cac-452d-89fc-7e18fb70658d jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -295,5 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: grammatical_between_1_2 reference: '' diff --git a/promptsource/templates/blimp/determiner_noun_agreement_with_adjective_1/templates.yaml b/promptsource/templates/blimp/determiner_noun_agreement_with_adjective_1/templates.yaml index 37945da63..ba83fe257 100644 --- a/promptsource/templates/blimp/determiner_noun_agreement_with_adjective_1/templates.yaml +++ b/promptsource/templates/blimp/determiner_noun_agreement_with_adjective_1/templates.yaml @@ -1,25 +1,9 @@ dataset: blimp subset: determiner_noun_agreement_with_adjective_1 templates: - 2b99daa2-9a73-44fb-b704-d28079ad0f3e: !Template + 177fdeaa-4de1-4e8f-a325-23ad33e86376: !Template answer_choices: Yes ||| No - id: 2b99daa2-9a73-44fb-b704-d28079ad0f3e - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes - reference: '' - 375d054d-246f-4ba2-af73-ec7b8a9d4681: !Template - answer_choices: Yes ||| No - id: 375d054d-246f-4ba2-af73-ec7b8a9d4681 + id: 177fdeaa-4de1-4e8f-a325-23ad33e86376 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -33,24 +17,24 @@ templates: original_task: false name: single_sentence_bad_yes_no reference: '' - 3da7b8e5-ddd2-4099-8045-8b3ca57bd3e2: !Template - answer_choices: 1 ||| 2 - id: 3da7b8e5-ddd2-4099-8045-8b3ca57bd3e2 + 33bfb629-a0c2-469e-b54d-571f9f874f46: !Template + answer_choices: A ||| B + id: 33bfb629-a0c2-469e-b54d-571f9f874f46 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -70,26 +54,26 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: grammatical_between_A_B reference: '' - 411946bc-ff32-41d0-afd9-0b9d9b87669d: !Template - answer_choices: 1 ||| 2 - id: 411946bc-ff32-41d0-afd9-0b9d9b87669d + 9b67bc74-7b46-4987-bf8f-9c0478eddd9c: !Template + answer_choices: A ||| B + id: 9b67bc74-7b46-4987-bf8f-9c0478eddd9c jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -109,30 +93,46 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: grammatical_which_one_A_B + reference: '' + a458d8d5-ca2f-4cf9-8ecf-00151c3c072e: !Template + answer_choices: Yes ||| No + id: a458d8d5-ca2f-4cf9-8ecf-00151c3c072e + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no reference: '' - 599dbe06-4c60-44e2-9fa0-63ce24c25191: !Template + a7e580a4-b5e3-42e2-879c-7a20bd58c64d: !Template answer_choices: A ||| B - id: 599dbe06-4c60-44e2-9fa0-63ce24c25191 + id: a7e580a4-b5e3-42e2-879c-7a20bd58c64d jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + Sentence A: {{ sentence_good }} Sentence B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} Sentence B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order + Which one is a better sentence of English, A or B? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -147,42 +147,26 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A - reference: '' - 688dd368-4fda-4442-8cf8-a9c324c6cdc4: !Template - answer_choices: Yes ||| No - id: 688dd368-4fda-4442-8cf8-a9c324c6cdc4 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no + name: better_A_B reference: '' - 8542d02a-7270-4eaf-b9df-ce7d0a36eb5d: !Template - answer_choices: A ||| B - id: 8542d02a-7270-4eaf-b9df-ce7d0a36eb5d + b1717065-b7f9-471d-ab1b-630c89bb0525: !Template + answer_choices: 1 ||| 2 + id: b1717065-b7f9-471d-ab1b-630c89bb0525 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -202,31 +186,32 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: grammatical_between_1_2 reference: '' - cf1c7c75-b761-453d-bdb6-8ad0b464b979: !Template - answer_choices: A ||| B - id: cf1c7c75-b761-453d-bdb6-8ad0b464b979 + c9aff3a4-145d-4e6a-91b0-bb482dd40856: !Template + answer_choices: 1 ||| 2 + id: c9aff3a4-145d-4e6a-91b0-bb482dd40856 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -240,32 +225,63 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: grammatical_which_one_1_2 + reference: '' + d0273898-368a-497f-bfde-e0625f3603fd: !Template + answer_choices: Yes ||| No + id: d0273898-368a-497f-bfde-e0625f3603fd + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes reference: '' - d64b6f4a-1759-4c71-aeef-ea164c474bba: !Template + f14441f0-315a-45e0-960d-b4ecbb6d661a: !Template + answer_choices: Yes ||| No + id: f14441f0-315a-45e0-960d-b4ecbb6d661a + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes + reference: '' + fb0d4859-3ca3-40cd-8328-8eb492283cfb: !Template answer_choices: A ||| B - id: d64b6f4a-1759-4c71-aeef-ea164c474bba + id: fb0d4859-3ca3-40cd-8328-8eb492283cfb jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -279,21 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B - reference: '' - e1dc6aa2-7fb4-4fa7-bb49-e5af7dd2c298: !Template - answer_choices: Yes ||| No - id: e1dc6aa2-7fb4-4fa7-bb49-e5af7dd2c298 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes + name: better_B_A reference: '' diff --git a/promptsource/templates/blimp/distractor_agreement_relational_noun/templates.yaml b/promptsource/templates/blimp/distractor_agreement_relational_noun/templates.yaml index b71d50866..df724fda3 100644 --- a/promptsource/templates/blimp/distractor_agreement_relational_noun/templates.yaml +++ b/promptsource/templates/blimp/distractor_agreement_relational_noun/templates.yaml @@ -1,13 +1,29 @@ dataset: blimp subset: distractor_agreement_relational_noun templates: - 2c5fb585-9573-4112-9e49-64b7d480bec2: !Template + 094bd674-442c-4613-8527-f4b908166fca: !Template answer_choices: Yes ||| No - id: 2c5fb585-9573-4112-9e49-64b7d480bec2 + id: 094bd674-442c-4613-8527-f4b908166fca jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 113bf501-a714-4a8d-a653-2f4b2f2f6ec3: !Template + answer_choices: Yes ||| No + id: 113bf501-a714-4a8d-a653-2f4b2f2f6ec3 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -15,26 +31,26 @@ templates: metrics: - Accuracy original_task: false - name: single_sentence_good_yes_no + name: single_sentence_good_no_yes reference: '' - 3e406e15-6f42-402a-8249-55362efe00c7: !Template - answer_choices: 1 ||| 2 - id: 3e406e15-6f42-402a-8249-55362efe00c7 + 1d8c3ab0-adc6-4f6d-8b6b-b612dae56f9d: !Template + answer_choices: A ||| B + id: 1d8c3ab0-adc6-4f6d-8b6b-b612dae56f9d jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -54,30 +70,14 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 - reference: '' - 3eccd02a-3fe5-4aff-b579-63d698e8bada: !Template - answer_choices: Yes ||| No - id: 3eccd02a-3fe5-4aff-b579-63d698e8bada - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes + name: grammatical_between_A_B reference: '' - 605639df-072a-47c6-b166-41824a867688: !Template + 4eef0ebe-6760-4a57-9492-2076915cadae: !Template answer_choices: A ||| B - id: 605639df-072a-47c6-b166-41824a867688 + id: 4eef0ebe-6760-4a57-9492-2076915cadae jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -109,11 +109,11 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: grammatical_which_one_A_B reference: '' - 8cf993bd-33db-41ee-92b3-1ed8132bdf84: !Template + 7fc778b4-aebe-40f3-b270-620630d7cde5: !Template answer_choices: Yes ||| No - id: 8cf993bd-33db-41ee-92b3-1ed8132bdf84 + id: 7fc778b4-aebe-40f3-b270-620630d7cde5 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -127,30 +127,29 @@ templates: original_task: false name: single_sentence_bad_no_yes reference: '' - 91569423-f8be-4f41-a740-fbadb9a3966e: !Template + a282f89a-c0d8-421b-bdbb-4fa0bb7d6a91: !Template answer_choices: A ||| B - id: 91569423-f8be-4f41-a740-fbadb9a3966e + id: a282f89a-c0d8-421b-bdbb-4fa0bb7d6a91 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -164,31 +163,32 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: better_A_B reference: '' - 9d474aca-b18a-4968-a5f5-dc16e4f79811: !Template - answer_choices: A ||| B - id: 9d474aca-b18a-4968-a5f5-dc16e4f79811 + bc3cb2f3-1ca6-42a8-81d0-8e36dd1cc13b: !Template + answer_choices: 1 ||| 2 + id: bc3cb2f3-1ca6-42a8-81d0-8e36dd1cc13b jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -202,24 +202,40 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: grammatical_between_1_2 reference: '' - dfe23dc0-6a82-4c40-9f3a-8340bf6fa7f1: !Template + c774c7b0-7868-45f9-a48c-3fae42eed917: !Template + answer_choices: Yes ||| No + id: c774c7b0-7868-45f9-a48c-3fae42eed917 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + c986eb7e-8e8f-46f3-b421-c18199d1ff1f: !Template answer_choices: A ||| B - id: dfe23dc0-6a82-4c40-9f3a-8340bf6fa7f1 + id: c986eb7e-8e8f-46f3-b421-c18199d1ff1f jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + Sentence A: {{ sentence_good }} Sentence B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} Sentence B: {{ sentence_good }} @@ -242,28 +258,12 @@ templates: original_task: true name: better_B_A reference: '' - eb9d8608-b55c-468a-8db5-790e87e09bd9: !Template - answer_choices: Yes ||| No - id: eb9d8608-b55c-468a-8db5-790e87e09bd9 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no - reference: '' - fd46b0be-1134-4cef-8ec3-a8807365683d: !Template + ddd82457-5ab4-41ec-a44e-d00be9bf3727: !Template answer_choices: 1 ||| 2 - id: fd46b0be-1134-4cef-8ec3-a8807365683d + id: ddd82457-5ab4-41ec-a44e-d00be9bf3727 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -295,5 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: grammatical_which_one_1_2 reference: '' diff --git a/promptsource/templates/blimp/distractor_agreement_relative_clause/templates.yaml b/promptsource/templates/blimp/distractor_agreement_relative_clause/templates.yaml index 1e525b66e..8046b7c8f 100644 --- a/promptsource/templates/blimp/distractor_agreement_relative_clause/templates.yaml +++ b/promptsource/templates/blimp/distractor_agreement_relative_clause/templates.yaml @@ -1,13 +1,29 @@ dataset: blimp subset: distractor_agreement_relative_clause templates: - 2a088be4-6e1b-420d-9668-e38584fce3df: !Template + 14c4b091-49b7-4c98-b37a-71af6ac4811c: !Template answer_choices: Yes ||| No - id: 2a088be4-6e1b-420d-9668-e38584fce3df + id: 14c4b091-49b7-4c98-b37a-71af6ac4811c jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes + reference: '' + 2f57ebd3-4617-46f4-96d7-afdacbd72469: !Template + answer_choices: Yes ||| No + id: 2f57ebd3-4617-46f4-96d7-afdacbd72469 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata @@ -15,32 +31,31 @@ templates: metrics: - Accuracy original_task: false - name: single_sentence_bad_yes_no + name: single_sentence_bad_no_yes reference: '' - 4c299217-6509-447e-80ee-c5834f0a18c4: !Template - answer_choices: 1 ||| 2 - id: 4c299217-6509-447e-80ee-c5834f0a18c4 + 4ad745ea-017b-41c5-a918-4c56d1daf41d: !Template + answer_choices: A ||| B + id: 4ad745ea-017b-41c5-a918-4c56d1daf41d jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -54,62 +69,69 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: better_A_B reference: '' - 55390108-07d1-4776-b73b-8ea179138065: !Template - answer_choices: Yes ||| No - id: 55390108-07d1-4776-b73b-8ea179138065 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + 4d1c247f-4d4c-4e1e-88cf-445eea5ea1e3: !Template + answer_choices: A ||| B + id: 4d1c247f-4d4c-4e1e-88cf-445eea5ea1e3 + jinja: '{% set shuffled_order = [0, 1] | random %} - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no - reference: '' - 87ea3119-81d0-4473-8e19-67c5e411670e: !Template - answer_choices: Yes ||| No - id: 87ea3119-81d0-4473-8e19-67c5e411670e - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + Which one of the following sentences is grammatical? Please answer A or B. - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: false - name: single_sentence_good_no_yes + original_task: true + name: grammatical_which_one_A_B reference: '' - 88793be0-3ce6-498a-a96e-a584e2dd0710: !Template + 55bfe218-6eab-40ee-a596-ec4b52956c83: !Template answer_choices: A ||| B - id: 88793be0-3ce6-498a-a96e-a584e2dd0710 + id: 55bfe218-6eab-40ee-a596-ec4b52956c83 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + Sentence A: {{ sentence_good }} Sentence B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} Sentence B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order + Which one is a better sentence of English, B or A? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -124,27 +146,27 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: better_B_A reference: '' - 9b0fe4e4-bbc5-4300-9b4c-f2c43928a009: !Template + 5768b427-5f4d-4b29-b89f-f4088fb47731: !Template answer_choices: Yes ||| No - id: 9b0fe4e4-bbc5-4300-9b4c-f2c43928a009 + id: 5768b427-5f4d-4b29-b89f-f4088fb47731 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_bad_no_yes + name: single_sentence_good_yes_no reference: '' - a48b6cdf-67b5-42ff-9a27-f672c061f6ee: !Template + 69563a61-ab33-4d02-92e2-04d848bdfdef: !Template answer_choices: 1 ||| 2 - id: a48b6cdf-67b5-42ff-9a27-f672c061f6ee + id: 69563a61-ab33-4d02-92e2-04d848bdfdef jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer 1 or 2. @@ -181,47 +203,25 @@ templates: original_task: true name: grammatical_which_one_1_2 reference: '' - c31818c9-00e8-4c3c-835a-64ab19feecae: !Template - answer_choices: A ||| B - id: c31818c9-00e8-4c3c-835a-64ab19feecae - jinja: '{% set shuffled_order = [0, 1] | random %} - - Read the following two sentences. - - {% if shuffled_order == 0 %} - - Sentence A: {{ sentence_good }} - - Sentence B: {{ sentence_bad }} - - {% else %} - - Sentence A: {{ sentence_bad }} - - Sentence B: {{ sentence_good }} - - {% endif %} - - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} + bffb193d-70a0-4790-abc2-40c98314019b: !Template + answer_choices: Yes ||| No + id: bffb193d-70a0-4790-abc2-40c98314019b + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - {% endif %}' + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: better_B_A + original_task: false + name: single_sentence_bad_yes_no reference: '' - f285dbe1-a96c-45d3-9efa-569e5e43d09c: !Template + cde5ffde-c1fb-4b8f-b52e-09ae2269a509: !Template answer_choices: A ||| B - id: f285dbe1-a96c-45d3-9efa-569e5e43d09c + id: cde5ffde-c1fb-4b8f-b52e-09ae2269a509 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer A or B. @@ -258,24 +258,24 @@ templates: original_task: true name: grammatical_between_A_B reference: '' - f506a776-19d1-4848-b311-257c2eeb1d36: !Template - answer_choices: A ||| B - id: f506a776-19d1-4848-b311-257c2eeb1d36 + fa717a13-e152-4131-925a-5a324aed53df: !Template + answer_choices: 1 ||| 2 + id: fa717a13-e152-4131-925a-5a324aed53df jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -295,5 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: grammatical_between_1_2 reference: '' diff --git a/promptsource/templates/blimp/drop_argument/templates.yaml b/promptsource/templates/blimp/drop_argument/templates.yaml index 20923279f..bca1a0de4 100644 --- a/promptsource/templates/blimp/drop_argument/templates.yaml +++ b/promptsource/templates/blimp/drop_argument/templates.yaml @@ -1,9 +1,9 @@ dataset: blimp subset: drop_argument templates: - 1de3a26c-989e-4842-a1b4-636a5d1422cc: !Template + 137031cb-2dcb-4881-86ea-7d221b84607c: !Template answer_choices: Yes ||| No - id: 1de3a26c-989e-4842-a1b4-636a5d1422cc + id: 137031cb-2dcb-4881-86ea-7d221b84607c jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -17,12 +17,12 @@ templates: original_task: false name: single_sentence_bad_yes_no reference: '' - 3cd42d1c-5cd2-4b9b-be35-5ed27fc2aabc: !Template + 1a04249a-4538-4830-8e7c-a516727c159d: !Template answer_choices: 1 ||| 2 - id: 3cd42d1c-5cd2-4b9b-be35-5ed27fc2aabc + id: 1a04249a-4538-4830-8e7c-a516727c159d jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -54,14 +54,14 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: grammatical_between_1_2 reference: '' - 4ee226bc-2b51-4cbe-8ee2-a54a376413ff: !Template + 263e3de0-32b1-4640-a6e1-2bca0deabdb1: !Template answer_choices: A ||| B - id: 4ee226bc-2b51-4cbe-8ee2-a54a376413ff + id: 263e3de0-32b1-4640-a6e1-2bca0deabdb1 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -93,42 +93,26 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B - reference: '' - 6a6296d1-b066-45f2-8550-915f8cc3a4be: !Template - answer_choices: Yes ||| No - id: 6a6296d1-b066-45f2-8550-915f8cc3a4be - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes + name: grammatical_which_one_A_B reference: '' - a79af759-f93a-4f7f-90a9-cf5391a7ca1a: !Template - answer_choices: 1 ||| 2 - id: a79af759-f93a-4f7f-90a9-cf5391a7ca1a + 34f51242-e535-4213-9f8f-7b54c54f64f1: !Template + answer_choices: A ||| B + id: 34f51242-e535-4213-9f8f-7b54c54f64f1 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -148,30 +132,30 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: grammatical_between_A_B reference: '' - afbb26fa-da68-4d2d-8333-63f7cadf9586: !Template + 3f6a0b47-5606-43f7-a08a-c6eb57d1c6d1: !Template answer_choices: A ||| B - id: afbb26fa-da68-4d2d-8333-63f7cadf9586 + id: 3f6a0b47-5606-43f7-a08a-c6eb57d1c6d1 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + Sentence A: {{ sentence_good }} Sentence B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} Sentence B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order + Which one is a better sentence of English, A or B? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -186,11 +170,11 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: better_A_B reference: '' - c0bdc602-91d0-40e9-af1e-d37b9b66d42f: !Template + 957ef5ba-fe78-48f3-9388-5e6f087b1eaf: !Template answer_choices: Yes ||| No - id: c0bdc602-91d0-40e9-af1e-d37b9b66d42f + id: 957ef5ba-fe78-48f3-9388-5e6f087b1eaf jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -204,24 +188,40 @@ templates: original_task: false name: single_sentence_bad_no_yes reference: '' - c4d081c0-7c66-48df-aa56-bc9cf089be76: !Template - answer_choices: A ||| B - id: c4d081c0-7c66-48df-aa56-bc9cf089be76 + 9c7271c8-ed8d-4850-b0fb-62116d7c90a7: !Template + answer_choices: Yes ||| No + id: 9c7271c8-ed8d-4850-b0fb-62116d7c90a7 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes + reference: '' + def99203-248c-4ff5-88c7-f40a6dd59c3d: !Template + answer_choices: 1 ||| 2 + id: def99203-248c-4ff5-88c7-f40a6dd59c3d jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -241,46 +241,30 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B - reference: '' - d84b674a-d72c-4bd0-8005-f2f3b4aed72f: !Template - answer_choices: Yes ||| No - id: d84b674a-d72c-4bd0-8005-f2f3b4aed72f - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no + name: grammatical_which_one_1_2 reference: '' - e18be316-c04c-4fa2-81b1-dba8e4c0305b: !Template + e2e39a00-9921-4068-a5d3-1d33e7afcefe: !Template answer_choices: A ||| B - id: e18be316-c04c-4fa2-81b1-dba8e4c0305b + id: e2e39a00-9921-4068-a5d3-1d33e7afcefe jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + Sentence A: {{ sentence_good }} Sentence B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} Sentence B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order + Which one is a better sentence of English, B or A? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -295,5 +279,21 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: better_B_A + reference: '' + ea2c6e6b-35c7-4cb1-b3dd-55638d32c856: !Template + answer_choices: Yes ||| No + id: ea2c6e6b-35c7-4cb1-b3dd-55638d32c856 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no reference: '' diff --git a/promptsource/templates/blimp/ellipsis_n_bar_1/templates.yaml b/promptsource/templates/blimp/ellipsis_n_bar_1/templates.yaml index c97d6cfc0..ac2ae42c2 100644 --- a/promptsource/templates/blimp/ellipsis_n_bar_1/templates.yaml +++ b/promptsource/templates/blimp/ellipsis_n_bar_1/templates.yaml @@ -1,30 +1,29 @@ dataset: blimp subset: ellipsis_n_bar_1 templates: - 061cf5f0-d23f-47b4-b481-bc3bf9d9abe6: !Template - answer_choices: 1 ||| 2 - id: 061cf5f0-d23f-47b4-b481-bc3bf9d9abe6 + 19c0226d-73dd-44fc-9884-23fa7b705705: !Template + answer_choices: A ||| B + id: 19c0226d-73dd-44fc-9884-23fa7b705705 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -38,26 +37,42 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: better_B_A reference: '' - 0629359c-ec2a-4853-b7d2-87cdfb65645d: !Template - answer_choices: 1 ||| 2 - id: 0629359c-ec2a-4853-b7d2-87cdfb65645d + 3deb1c9e-7141-4aea-b88c-3573b88df02b: !Template + answer_choices: Yes ||| No + id: 3deb1c9e-7141-4aea-b88c-3573b88df02b + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 50c8864d-8d10-41dc-b77d-355f80e5c45b: !Template + answer_choices: A ||| B + id: 50c8864d-8d10-41dc-b77d-355f80e5c45b jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -77,11 +92,27 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: grammatical_between_A_B + reference: '' + a060f971-64cd-40f7-98dc-f7d369877941: !Template + answer_choices: Yes ||| No + id: a060f971-64cd-40f7-98dc-f7d369877941 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes reference: '' - 253d1bc9-dec2-4173-a047-6714ee4716b0: !Template + bb0a0195-7b97-4814-b301-2649bf529be6: !Template answer_choices: Yes ||| No - id: 253d1bc9-dec2-4173-a047-6714ee4716b0 + id: bb0a0195-7b97-4814-b301-2649bf529be6 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -95,28 +126,28 @@ templates: original_task: false name: single_sentence_bad_no_yes reference: '' - 34909b4a-ad88-4899-a05a-9e2465a790f8: !Template + c9b18356-a2e6-445f-a595-c056cbda3247: !Template answer_choices: A ||| B - id: 34909b4a-ad88-4899-a05a-9e2465a790f8 + id: c9b18356-a2e6-445f-a595-c056cbda3247 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + Sentence A: {{ sentence_good }} Sentence B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} Sentence B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order + Which one is a better sentence of English, A or B? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -131,43 +162,11 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A - reference: '' - 3ea2c661-b45e-42a1-b962-ca9cef661ae1: !Template - answer_choices: Yes ||| No - id: 3ea2c661-b45e-42a1-b962-ca9cef661ae1 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes - reference: '' - 681bb1fd-a79e-403d-830d-1a138293741e: !Template - answer_choices: Yes ||| No - id: 681bb1fd-a79e-403d-830d-1a138293741e - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no + name: better_A_B reference: '' - 8e1bcbb0-ff9a-4b00-a325-dbb758681043: !Template + caaafbb1-f55a-4010-ad6d-6a6c34d8ac97: !Template answer_choices: A ||| B - id: 8e1bcbb0-ff9a-4b00-a325-dbb758681043 + id: caaafbb1-f55a-4010-ad6d-6a6c34d8ac97 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer A or B. @@ -204,29 +203,30 @@ templates: original_task: true name: grammatical_which_one_A_B reference: '' - a45b964c-c296-46b7-87ec-4a3d7d8f9098: !Template - answer_choices: A ||| B - id: a45b964c-c296-46b7-87ec-4a3d7d8f9098 + cb59eede-b5c3-450f-ac87-ce35bcd6f4f5: !Template + answer_choices: 1 ||| 2 + id: cb59eede-b5c3-450f-ac87-ce35bcd6f4f5 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -240,42 +240,42 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: grammatical_between_1_2 reference: '' - c4ee8fa5-2b8a-44de-a630-9794cc6dcc51: !Template + d3196a24-e005-422b-a9a4-d5d6327751fa: !Template answer_choices: Yes ||| No - id: c4ee8fa5-2b8a-44de-a630-9794cc6dcc51 + id: d3196a24-e005-422b-a9a4-d5d6327751fa jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_bad_yes_no + name: single_sentence_good_yes_no reference: '' - de9f89ef-d20d-45c9-a22c-3f229a465094: !Template - answer_choices: A ||| B - id: de9f89ef-d20d-45c9-a22c-3f229a465094 + fcd5cc9e-fd01-46d2-97b0-8147d23e3514: !Template + answer_choices: 1 ||| 2 + id: fcd5cc9e-fd01-46d2-97b0-8147d23e3514 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -295,5 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: grammatical_which_one_1_2 reference: '' diff --git a/promptsource/templates/blimp/ellipsis_n_bar_2/templates.yaml b/promptsource/templates/blimp/ellipsis_n_bar_2/templates.yaml index 297b5d886..83b3175f6 100644 --- a/promptsource/templates/blimp/ellipsis_n_bar_2/templates.yaml +++ b/promptsource/templates/blimp/ellipsis_n_bar_2/templates.yaml @@ -1,46 +1,45 @@ dataset: blimp subset: ellipsis_n_bar_2 templates: - 02cf9b03-67fc-414a-b3f1-bcdd0f65929b: !Template + 3b95807f-c6e0-483e-a32b-d564c754df56: !Template answer_choices: Yes ||| No - id: 02cf9b03-67fc-414a-b3f1-bcdd0f65929b + id: 3b95807f-c6e0-483e-a32b-d564c754df56 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_bad_yes_no + name: single_sentence_good_no_yes reference: '' - 08c26355-4c8b-4bf6-95b0-16025b65bd25: !Template + 4537d356-e7e6-46c7-acff-a1b26ca23af9: !Template answer_choices: A ||| B - id: 08c26355-4c8b-4bf6-95b0-16025b65bd25 + id: 4537d356-e7e6-46c7-acff-a1b26ca23af9 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -54,11 +53,27 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: better_B_A reference: '' - 25e742f5-a100-4f06-a1eb-a9203afa8427: !Template + 4eb6609d-8070-4840-a423-d486b2f524ce: !Template answer_choices: Yes ||| No - id: 25e742f5-a100-4f06-a1eb-a9203afa8427 + id: 4eb6609d-8070-4840-a423-d486b2f524ce + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes + reference: '' + 5db7087e-8f5a-46b6-bc65-975e677e9220: !Template + answer_choices: Yes ||| No + id: 5db7087e-8f5a-46b6-bc65-975e677e9220 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -72,9 +87,9 @@ templates: original_task: false name: single_sentence_good_yes_no reference: '' - 2f68abe6-8abc-4e7e-ad3f-af512ec8e7b6: !Template + 5f6ee442-9961-4f8c-ba36-32a63ec86239: !Template answer_choices: 1 ||| 2 - id: 2f68abe6-8abc-4e7e-ad3f-af512ec8e7b6 + id: 5f6ee442-9961-4f8c-ba36-32a63ec86239 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer 1 or 2. @@ -111,40 +126,24 @@ templates: original_task: true name: grammatical_which_one_1_2 reference: '' - 4245a98b-a715-47b1-ac54-9f8eb344840b: !Template - answer_choices: Yes ||| No - id: 4245a98b-a715-47b1-ac54-9f8eb344840b - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes - reference: '' - 5a8b5b95-b762-41a4-8367-6e28f4f1eeef: !Template - answer_choices: A ||| B - id: 5a8b5b95-b762-41a4-8367-6e28f4f1eeef + 85c6d2a7-e090-42e8-be22-eb22350aa724: !Template + answer_choices: 1 ||| 2 + id: 85c6d2a7-e090-42e8-be22-eb22350aa724 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -164,24 +163,24 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: grammatical_between_1_2 reference: '' - 5d6c9179-d290-40f4-b0e9-9446182f14cb: !Template + 8e1c6033-6525-410b-b94e-bd7a77b7fe60: !Template answer_choices: A ||| B - id: 5d6c9179-d290-40f4-b0e9-9446182f14cb + id: 8e1c6033-6525-410b-b94e-bd7a77b7fe60 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + Sentence A: {{ sentence_good }} Sentence B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} Sentence B: {{ sentence_good }} @@ -204,24 +203,24 @@ templates: original_task: true name: better_A_B reference: '' - 67f471fd-ebf0-44b2-92e5-4775a16cfd5a: !Template - answer_choices: 1 ||| 2 - id: 67f471fd-ebf0-44b2-92e5-4775a16cfd5a + aaca0df7-7c36-4bb2-ba51-fe083881769c: !Template + answer_choices: A ||| B + id: aaca0df7-7c36-4bb2-ba51-fe083881769c jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -241,31 +240,32 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: grammatical_which_one_A_B reference: '' - a6ce1219-883b-4af2-ab8c-223adca927d9: !Template + abe0fa7e-b84f-44b2-91c2-9ba8f64d18c8: !Template answer_choices: A ||| B - id: a6ce1219-883b-4af2-ab8c-223adca927d9 + id: abe0fa7e-b84f-44b2-91c2-9ba8f64d18c8 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -279,21 +279,21 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: grammatical_between_A_B reference: '' - e51c0a4e-f7bc-4f10-b3c4-161b5e8abd32: !Template + d0cc6ba7-4cfe-458b-acab-3da34a1ef42e: !Template answer_choices: Yes ||| No - id: e51c0a4e-f7bc-4f10-b3c4-161b5e8abd32 + id: d0cc6ba7-4cfe-458b-acab-3da34a1ef42e jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_good_no_yes + name: single_sentence_bad_yes_no reference: '' diff --git a/promptsource/templates/blimp/existential_there_object_raising/templates.yaml b/promptsource/templates/blimp/existential_there_object_raising/templates.yaml index 57f65c390..24b8b3c50 100644 --- a/promptsource/templates/blimp/existential_there_object_raising/templates.yaml +++ b/promptsource/templates/blimp/existential_there_object_raising/templates.yaml @@ -1,45 +1,30 @@ dataset: blimp subset: existential_there_object_raising templates: - 1181c64d-a4c9-48ee-a3d7-a0387d1716e1: !Template - answer_choices: Yes ||| No - id: 1181c64d-a4c9-48ee-a3d7-a0387d1716e1 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no - reference: '' - 25cf67cc-c980-4e7a-9ef0-1f860f2813b8: !Template - answer_choices: A ||| B - id: 25cf67cc-c980-4e7a-9ef0-1f860f2813b8 + 05b0f0ab-68b3-4280-99e6-7d721cfe1d17: !Template + answer_choices: 1 ||| 2 + id: 05b0f0ab-68b3-4280-99e6-7d721cfe1d17 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -53,26 +38,26 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: grammatical_which_one_1_2 reference: '' - 278cc4e0-1c44-4786-828b-52858e29ee66: !Template - answer_choices: 1 ||| 2 - id: 278cc4e0-1c44-4786-828b-52858e29ee66 + 59e89611-a575-403c-adb4-e703f578f618: !Template + answer_choices: A ||| B + id: 59e89611-a575-403c-adb4-e703f578f618 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -92,11 +77,11 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: grammatical_between_A_B reference: '' - 2f909ebc-9ded-4fb3-818a-6be2c000e9d6: !Template + 5fe1eba0-f56a-4faf-b87e-5bfd67dbfd37: !Template answer_choices: A ||| B - id: 2f909ebc-9ded-4fb3-818a-6be2c000e9d6 + id: 5fe1eba0-f56a-4faf-b87e-5bfd67dbfd37 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer A or B. @@ -133,13 +118,29 @@ templates: original_task: true name: grammatical_which_one_A_B reference: '' - 3f62cc54-edab-4d7b-93ee-6f716464c5d3: !Template + 62b5bce1-fcc6-475c-a424-c38fa7827dc8: !Template answer_choices: Yes ||| No - id: 3f62cc54-edab-4d7b-93ee-6f716464c5d3 + id: 62b5bce1-fcc6-475c-a424-c38fa7827dc8 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes + reference: '' + 676499fa-0650-4a98-ace8-b3b10423b719: !Template + answer_choices: Yes ||| No + id: 676499fa-0650-4a98-ace8-b3b10423b719 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata @@ -147,26 +148,26 @@ templates: metrics: - Accuracy original_task: false - name: single_sentence_bad_no_yes + name: single_sentence_bad_yes_no reference: '' - 6f37b12f-0176-46a4-93b5-4432b3a8ccb2: !Template - answer_choices: A ||| B - id: 6f37b12f-0176-46a4-93b5-4432b3a8ccb2 + 69dcb136-2b75-49ea-9256-9033c454f3f3: !Template + answer_choices: 1 ||| 2 + id: 69dcb136-2b75-49ea-9256-9033c454f3f3 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -186,48 +187,31 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B - reference: '' - 7a8e4d4d-2b2a-4890-bb57-34c33ee66e15: !Template - answer_choices: Yes ||| No - id: 7a8e4d4d-2b2a-4890-bb57-34c33ee66e15 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no + name: grammatical_between_1_2 reference: '' - 95c3afa9-20b9-479f-bf26-9d4f46c03b1b: !Template - answer_choices: 1 ||| 2 - id: 95c3afa9-20b9-479f-bf26-9d4f46c03b1b + 82f47664-d17b-493a-a009-6ef086ed9510: !Template + answer_choices: A ||| B + id: 82f47664-d17b-493a-a009-6ef086ed9510 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -241,46 +225,30 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 - reference: '' - a64d94c9-9b06-4a4c-83bd-c07f49969c9e: !Template - answer_choices: Yes ||| No - id: a64d94c9-9b06-4a4c-83bd-c07f49969c9e - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes + name: better_B_A reference: '' - dbcf12cd-3dad-499e-9f67-bb9cf496842f: !Template + 91a5714e-0948-4930-b6fb-05a3a733f80e: !Template answer_choices: A ||| B - id: dbcf12cd-3dad-499e-9f67-bb9cf496842f + id: 91a5714e-0948-4930-b6fb-05a3a733f80e jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + Sentence A: {{ sentence_good }} Sentence B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} Sentence B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order + Which one is a better sentence of English, A or B? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -295,5 +263,37 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: better_A_B + reference: '' + a7640552-1d97-4f0f-90ea-5f4002d150f0: !Template + answer_choices: Yes ||| No + id: a7640552-1d97-4f0f-90ea-5f4002d150f0 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + bb421227-948f-47c4-907c-542b98795b65: !Template + answer_choices: Yes ||| No + id: bb421227-948f-47c4-907c-542b98795b65 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes reference: '' diff --git a/promptsource/templates/blimp/existential_there_quantifiers_1/templates.yaml b/promptsource/templates/blimp/existential_there_quantifiers_1/templates.yaml index b2ee9a440..7552a3d2f 100644 --- a/promptsource/templates/blimp/existential_there_quantifiers_1/templates.yaml +++ b/promptsource/templates/blimp/existential_there_quantifiers_1/templates.yaml @@ -1,9 +1,9 @@ dataset: blimp subset: existential_there_quantifiers_1 templates: - 0ed94cec-4972-4d68-a41c-13cd3ae69a0b: !Template + 1360ba62-2cd9-4bd2-ac31-eb0c01d8b746: !Template answer_choices: A ||| B - id: 0ed94cec-4972-4d68-a41c-13cd3ae69a0b + id: 1360ba62-2cd9-4bd2-ac31-eb0c01d8b746 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer A or B. @@ -40,30 +40,29 @@ templates: original_task: true name: grammatical_which_one_A_B reference: '' - 374e8fdf-e845-4858-80c6-91de64c48954: !Template - answer_choices: 1 ||| 2 - id: 374e8fdf-e845-4858-80c6-91de64c48954 + 4371ddfa-c506-4428-8a5b-72b93d0c95f2: !Template + answer_choices: A ||| B + id: 4371ddfa-c506-4428-8a5b-72b93d0c95f2 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -77,47 +76,32 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 - reference: '' - 3a86efed-1f19-4236-9623-f7c9913ff79a: !Template - answer_choices: Yes ||| No - id: 3a86efed-1f19-4236-9623-f7c9913ff79a - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no + name: better_A_B reference: '' - 50a44f93-2e19-4e1e-8a16-f421ca6f8d36: !Template + 666d69fb-9b69-46ea-9fee-9edb7314a796: !Template answer_choices: A ||| B - id: 50a44f93-2e19-4e1e-8a16-f421ca6f8d36 + id: 666d69fb-9b69-46ea-9fee-9edb7314a796 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -131,11 +115,11 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: grammatical_between_A_B reference: '' - 762f5547-0b18-4b1c-a66c-a5b2c1dbdb07: !Template + 67c2cc5f-809a-4729-b860-15ee05624695: !Template answer_choices: 1 ||| 2 - id: 762f5547-0b18-4b1c-a66c-a5b2c1dbdb07 + id: 67c2cc5f-809a-4729-b860-15ee05624695 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer 1 or 2. @@ -172,24 +156,56 @@ templates: original_task: true name: grammatical_which_one_1_2 reference: '' - 94ddff00-ce06-41a3-86df-044b57b9c989: !Template - answer_choices: A ||| B - id: 94ddff00-ce06-41a3-86df-044b57b9c989 + 6fa18a95-644d-4587-8d9a-2c756e816250: !Template + answer_choices: Yes ||| No + id: 6fa18a95-644d-4587-8d9a-2c756e816250 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 7656ed79-c62c-4f0c-bb5e-4681c27f3f6c: !Template + answer_choices: Yes ||| No + id: 7656ed79-c62c-4f0c-bb5e-4681c27f3f6c + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + a87f84d3-d52f-4b70-9b2c-a71fb46e3019: !Template + answer_choices: 1 ||| 2 + id: a87f84d3-d52f-4b70-9b2c-a71fb46e3019 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -209,46 +225,46 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: grammatical_between_1_2 reference: '' - ceaa3f86-04e5-4e53-a007-1a40994629f0: !Template + d0e8b05c-5ff1-4e31-b155-e63d55c41cd5: !Template answer_choices: Yes ||| No - id: ceaa3f86-04e5-4e53-a007-1a40994629f0 + id: d0e8b05c-5ff1-4e31-b155-e63d55c41cd5 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_bad_yes_no + name: single_sentence_good_no_yes reference: '' - d39ea036-c626-4487-88be-48a168a99726: !Template + e039f4ee-f93b-4572-babf-b1cd29bed2c5: !Template answer_choices: A ||| B - id: d39ea036-c626-4487-88be-48a168a99726 + id: e039f4ee-f93b-4572-babf-b1cd29bed2c5 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + Sentence A: {{ sentence_good }} Sentence B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} Sentence B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order + Which one is a better sentence of English, B or A? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -263,11 +279,11 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: better_B_A reference: '' - e8239a92-911a-4a48-b5da-60507c0e6081: !Template + eae69c1e-3809-46ef-b9b8-5a6eb8a07c1d: !Template answer_choices: Yes ||| No - id: e8239a92-911a-4a48-b5da-60507c0e6081 + id: eae69c1e-3809-46ef-b9b8-5a6eb8a07c1d jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -281,19 +297,3 @@ templates: original_task: false name: single_sentence_bad_no_yes reference: '' - fac46819-b60e-4694-be22-3074d3d914a0: !Template - answer_choices: Yes ||| No - id: fac46819-b60e-4694-be22-3074d3d914a0 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes - reference: '' diff --git a/promptsource/templates/blimp/existential_there_quantifiers_2/templates.yaml b/promptsource/templates/blimp/existential_there_quantifiers_2/templates.yaml index e5c5486f9..87d5c6794 100644 --- a/promptsource/templates/blimp/existential_there_quantifiers_2/templates.yaml +++ b/promptsource/templates/blimp/existential_there_quantifiers_2/templates.yaml @@ -1,61 +1,85 @@ dataset: blimp subset: existential_there_quantifiers_2 templates: - 1f21002e-fb16-4aa7-82c1-449dc8c2da47: !Template - answer_choices: Yes ||| No - id: 1f21002e-fb16-4aa7-82c1-449dc8c2da47 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + 344a1e36-e545-4446-a016-519b4b1fe605: !Template + answer_choices: 1 ||| 2 + id: 344a1e36-e545-4446-a016-519b4b1fe605 + jinja: '{% set shuffled_order = [0, 1] | random %} - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: false - name: single_sentence_good_yes_no + original_task: true + name: grammatical_which_one_1_2 reference: '' - 20661c62-c2e1-4543-8248-4a4674448cbc: !Template + 451b7268-d0ea-4e09-a010-b87a59a90a09: !Template answer_choices: Yes ||| No - id: 20661c62-c2e1-4543-8248-4a4674448cbc + id: 451b7268-d0ea-4e09-a010-b87a59a90a09 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_good_no_yes + name: single_sentence_bad_yes_no reference: '' - 342eb8f1-a729-4834-aae1-ccf6972f0529: !Template + 4ff8ebc7-c939-4142-bfb7-c5f1d02b1a45: !Template answer_choices: A ||| B - id: 342eb8f1-a729-4834-aae1-ccf6972f0529 + id: 4ff8ebc7-c939-4142-bfb7-c5f1d02b1a45 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -69,14 +93,14 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: grammatical_between_A_B reference: '' - 6cc4b292-85f5-43b5-acc1-d84e8345ae5e: !Template + 888ab7df-512d-416d-ad3a-5a26d3d1f072: !Template answer_choices: A ||| B - id: 6cc4b292-85f5-43b5-acc1-d84e8345ae5e + id: 888ab7df-512d-416d-ad3a-5a26d3d1f072 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -108,15 +132,15 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: grammatical_which_one_A_B reference: '' - 86521763-1a17-4c88-a6d7-5b9e5b6bba2c: !Template + 925ec313-6b21-47e5-ba43-c3b530549f40: !Template answer_choices: Yes ||| No - id: 86521763-1a17-4c88-a6d7-5b9e5b6bba2c + id: 925ec313-6b21-47e5-ba43-c3b530549f40 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata @@ -124,53 +148,46 @@ templates: metrics: - Accuracy original_task: false - name: single_sentence_bad_yes_no + name: single_sentence_bad_no_yes reference: '' - 8b069711-571b-4b45-b050-7ea4b0a3d472: !Template - answer_choices: 1 ||| 2 - id: 8b069711-571b-4b45-b050-7ea4b0a3d472 - jinja: '{% set shuffled_order = [0, 1] | random %} - - Between the two sentences, which one is grammatical? Please answer 1 or 2. - - {% if shuffled_order == 0 %} - - 1: {{ sentence_good }} - - 2: {{ sentence_bad }} - - {% else %} - - 1: {{ sentence_bad }} - - 2: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - {{ answer_choices[0] }} - - {% else %} + 94d87926-20ed-4e60-936b-7b2be9d498f5: !Template + answer_choices: Yes ||| No + id: 94d87926-20ed-4e60-936b-7b2be9d498f5 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - {{ answer_choices[1] }} + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes + reference: '' + d58c25a5-ca52-4adc-874d-a8e80fc71baa: !Template + answer_choices: Yes ||| No + id: d58c25a5-ca52-4adc-874d-a8e80fc71baa + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - {% endif %}' + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: grammatical_between_1_2 + original_task: false + name: single_sentence_good_yes_no reference: '' - ad7120e1-87f2-4144-a900-ba99b07364c5: !Template + d76b260b-90bc-42de-b23a-dac7449d8e67: !Template answer_choices: 1 ||| 2 - id: ad7120e1-87f2-4144-a900-ba99b07364c5 + id: d76b260b-90bc-42de-b23a-dac7449d8e67 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -202,40 +219,24 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 - reference: '' - b1afcf7c-0086-41bb-a892-627aeedd27dc: !Template - answer_choices: Yes ||| No - id: b1afcf7c-0086-41bb-a892-627aeedd27dc - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes + name: grammatical_between_1_2 reference: '' - c5cf39d6-a801-4ec8-840d-f569af4786b5: !Template + ec7c4445-1cc6-4e59-b1f7-40ebb2362670: !Template answer_choices: A ||| B - id: c5cf39d6-a801-4ec8-840d-f569af4786b5 + id: ec7c4445-1cc6-4e59-b1f7-40ebb2362670 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + Sentence A: {{ sentence_good }} Sentence B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} Sentence B: {{ sentence_good }} @@ -258,30 +259,29 @@ templates: original_task: true name: better_A_B reference: '' - e8338a6f-2fd3-4aac-8891-b70553f7eb04: !Template + f074d38b-9b01-4641-8c49-237978c8358a: !Template answer_choices: A ||| B - id: e8338a6f-2fd3-4aac-8891-b70553f7eb04 + id: f074d38b-9b01-4641-8c49-237978c8358a jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -295,5 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: better_B_A reference: '' diff --git a/promptsource/templates/blimp/existential_there_subject_raising/templates.yaml b/promptsource/templates/blimp/existential_there_subject_raising/templates.yaml index 18485c6f5..ff24dceae 100644 --- a/promptsource/templates/blimp/existential_there_subject_raising/templates.yaml +++ b/promptsource/templates/blimp/existential_there_subject_raising/templates.yaml @@ -1,29 +1,46 @@ dataset: blimp subset: existential_there_subject_raising templates: - 370bd524-e7f2-4fe7-8191-db3b32cc6e20: !Template + 081bd04d-338a-4875-9a3c-0779ff7b58fd: !Template + answer_choices: Yes ||| No + id: 081bd04d-338a-4875-9a3c-0779ff7b58fd + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes + reference: '' + 100e59a9-57ad-4086-b4e0-a7b401e81439: !Template answer_choices: A ||| B - id: 370bd524-e7f2-4fe7-8191-db3b32cc6e20 + id: 100e59a9-57ad-4086-b4e0-a7b401e81439 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -37,26 +54,26 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: grammatical_which_one_A_B reference: '' - 3fbce7f4-5596-4ed1-9aec-e23e85738cfa: !Template - answer_choices: A ||| B - id: 3fbce7f4-5596-4ed1-9aec-e23e85738cfa + 29073d22-1639-42fb-b95f-057f42fa6cfe: !Template + answer_choices: 1 ||| 2 + id: 29073d22-1639-42fb-b95f-057f42fa6cfe jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -76,48 +93,31 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B - reference: '' - 59581add-819b-496b-8d61-cf373f163567: !Template - answer_choices: Yes ||| No - id: 59581add-819b-496b-8d61-cf373f163567 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes + name: grammatical_which_one_1_2 reference: '' - 71e38631-705c-47b8-a836-b8ab2e9c2329: !Template + 5ddd7407-a777-45c4-8db3-3efc77cf80b7: !Template answer_choices: A ||| B - id: 71e38631-705c-47b8-a836-b8ab2e9c2329 + id: 5ddd7407-a777-45c4-8db3-3efc77cf80b7 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -131,11 +131,27 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: better_B_A + reference: '' + 69c565d8-e48a-4ab8-ba9a-c42936192efb: !Template + answer_choices: Yes ||| No + id: 69c565d8-e48a-4ab8-ba9a-c42936192efb + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no reference: '' - 7bb5c9ac-5c34-4c72-a273-981c13d03ef8: !Template + ae7f1f09-f621-4adc-8011-6b0ad7ca9bdc: !Template answer_choices: Yes ||| No - id: 7bb5c9ac-5c34-4c72-a273-981c13d03ef8 + id: ae7f1f09-f621-4adc-8011-6b0ad7ca9bdc jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -149,30 +165,29 @@ templates: original_task: false name: single_sentence_bad_yes_no reference: '' - 84a0b0ee-a021-4feb-833a-c982c6e55150: !Template - answer_choices: 1 ||| 2 - id: 84a0b0ee-a021-4feb-833a-c982c6e55150 + b01e7768-6ca7-45ee-bc88-e486c616760e: !Template + answer_choices: A ||| B + id: b01e7768-6ca7-45ee-bc88-e486c616760e jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -186,14 +201,30 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: better_A_B reference: '' - 88ecc6bc-9887-410a-bf59-426ebf0a9d52: !Template + c52a261b-fef4-4f3d-8d8e-f8372eec8e0b: !Template + answer_choices: Yes ||| No + id: c52a261b-fef4-4f3d-8d8e-f8372eec8e0b + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes + reference: '' + ca577db0-28ac-4b80-8e26-837772f5a8e8: !Template answer_choices: 1 ||| 2 - id: 88ecc6bc-9887-410a-bf59-426ebf0a9d52 + id: ca577db0-28ac-4b80-8e26-837772f5a8e8 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -225,31 +256,32 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: grammatical_between_1_2 reference: '' - 8ec321ed-96f6-4c86-83b2-e2b6aa3688af: !Template + de92564e-4bfc-4799-95e9-1e5b1693ba53: !Template answer_choices: A ||| B - id: 8ec321ed-96f6-4c86-83b2-e2b6aa3688af + id: de92564e-4bfc-4799-95e9-1e5b1693ba53 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -263,37 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A - reference: '' - a43ac65c-c607-428d-8603-45154cff7d93: !Template - answer_choices: Yes ||| No - id: a43ac65c-c607-428d-8603-45154cff7d93 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no - reference: '' - ae4a0b4a-dc6f-473c-9fb9-9c9d0a35f722: !Template - answer_choices: Yes ||| No - id: ae4a0b4a-dc6f-473c-9fb9-9c9d0a35f722 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes + name: grammatical_between_A_B reference: '' diff --git a/promptsource/templates/blimp/expletive_it_object_raising/templates.yaml b/promptsource/templates/blimp/expletive_it_object_raising/templates.yaml index f342637bc..c0f37c21f 100644 --- a/promptsource/templates/blimp/expletive_it_object_raising/templates.yaml +++ b/promptsource/templates/blimp/expletive_it_object_raising/templates.yaml @@ -1,24 +1,24 @@ dataset: blimp subset: expletive_it_object_raising templates: - 0401f307-fd12-4067-9f1d-ba3cbe169835: !Template - answer_choices: A ||| B - id: 0401f307-fd12-4067-9f1d-ba3cbe169835 + 2738759e-e8b1-4601-8adb-c8d6d018cc1e: !Template + answer_choices: 1 ||| 2 + id: 2738759e-e8b1-4601-8adb-c8d6d018cc1e jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -38,11 +38,27 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: grammatical_which_one_1_2 + reference: '' + 2b7cee78-c986-43f3-b96f-7410e466b14f: !Template + answer_choices: Yes ||| No + id: 2b7cee78-c986-43f3-b96f-7410e466b14f + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes reference: '' - 05627b8a-cb03-4ee6-9a5d-8580468b6750: !Template + 3a093a67-8a99-4382-b040-84343772a821: !Template answer_choices: A ||| B - id: 05627b8a-cb03-4ee6-9a5d-8580468b6750 + id: 3a093a67-8a99-4382-b040-84343772a821 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer A or B. @@ -79,29 +95,46 @@ templates: original_task: true name: grammatical_which_one_A_B reference: '' - 07d86f3e-935a-42f7-bf65-efb6e528ff7e: !Template + 6f3422f5-2fdc-4747-95c2-1f8c6114e735: !Template + answer_choices: Yes ||| No + id: 6f3422f5-2fdc-4747-95c2-1f8c6114e735 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 76b00939-25a0-4ca9-aa0c-7ef73f05c922: !Template answer_choices: A ||| B - id: 07d86f3e-935a-42f7-bf65-efb6e528ff7e + id: 76b00939-25a0-4ca9-aa0c-7ef73f05c922 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -115,48 +148,31 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B - reference: '' - 10be0bc5-5081-4c9e-8e05-f6d291d72aed: !Template - answer_choices: Yes ||| No - id: 10be0bc5-5081-4c9e-8e05-f6d291d72aed - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes + name: grammatical_between_A_B reference: '' - 190d288d-a1f0-4013-b89d-cd7024eac333: !Template - answer_choices: 1 ||| 2 - id: 190d288d-a1f0-4013-b89d-cd7024eac333 + 7c662985-1117-43a8-a085-89fb33695c32: !Template + answer_choices: A ||| B + id: 7c662985-1117-43a8-a085-89fb33695c32 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -170,11 +186,27 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: better_B_A reference: '' - 22537da5-d41b-4a0a-ba6b-5a169615ebca: !Template + 9bda7c86-3f6c-45d8-a5ce-29a79a68bb4b: !Template answer_choices: Yes ||| No - id: 22537da5-d41b-4a0a-ba6b-5a169615ebca + id: 9bda7c86-3f6c-45d8-a5ce-29a79a68bb4b + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes + reference: '' + a38b698a-268c-49e8-9a6f-3eea14c6daef: !Template + answer_choices: Yes ||| No + id: a38b698a-268c-49e8-9a6f-3eea14c6daef jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -188,9 +220,9 @@ templates: original_task: false name: single_sentence_good_yes_no reference: '' - 36ada838-12df-4a90-8129-c163a16f23b8: !Template + a59375ab-3c07-4678-8b1c-53acb31357ef: !Template answer_choices: 1 ||| 2 - id: 36ada838-12df-4a90-8129-c163a16f23b8 + id: a59375ab-3c07-4678-8b1c-53acb31357ef jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -227,44 +259,28 @@ templates: original_task: true name: grammatical_between_1_2 reference: '' - b9c464ed-634c-426e-bbd3-43b99a790110: !Template - answer_choices: Yes ||| No - id: b9c464ed-634c-426e-bbd3-43b99a790110 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no - reference: '' - f0de1f54-d371-4e1b-af4e-ea9c83cf0226: !Template + cf136e9f-e1f6-47f4-91c2-4ca1992a22c9: !Template answer_choices: A ||| B - id: f0de1f54-d371-4e1b-af4e-ea9c83cf0226 + id: cf136e9f-e1f6-47f4-91c2-4ca1992a22c9 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + Sentence A: {{ sentence_good }} Sentence B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} Sentence B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order + Which one is a better sentence of English, A or B? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -279,21 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A - reference: '' - f85dad10-90f2-427a-887a-d2fb243a54b5: !Template - answer_choices: Yes ||| No - id: f85dad10-90f2-427a-887a-d2fb243a54b5 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes + name: better_A_B reference: '' diff --git a/promptsource/templates/blimp/inchoative/templates.yaml b/promptsource/templates/blimp/inchoative/templates.yaml index 0aca8bbea..499d26916 100644 --- a/promptsource/templates/blimp/inchoative/templates.yaml +++ b/promptsource/templates/blimp/inchoative/templates.yaml @@ -1,30 +1,29 @@ dataset: blimp subset: inchoative templates: - 35ce6b18-a935-4b84-9665-f0e7e5b28591: !Template + 218b06da-2d74-429d-af75-97a69d9a2919: !Template answer_choices: A ||| B - id: 35ce6b18-a935-4b84-9665-f0e7e5b28591 + id: 218b06da-2d74-429d-af75-97a69d9a2919 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -38,11 +37,11 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: better_B_A reference: '' - 39734e83-f9d4-4fd1-aaff-56e218bf7406: !Template + 58ec605f-984b-4907-bf95-7fa933b2bffb: !Template answer_choices: 1 ||| 2 - id: 39734e83-f9d4-4fd1-aaff-56e218bf7406 + id: 58ec605f-984b-4907-bf95-7fa933b2bffb jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -79,9 +78,57 @@ templates: original_task: true name: grammatical_between_1_2 reference: '' - 7aadbc5d-c690-4ed1-a85a-06634ee52a1c: !Template + 67f86c5c-16e5-47be-bd77-5cd471b7c917: !Template + answer_choices: Yes ||| No + id: 67f86c5c-16e5-47be-bd77-5cd471b7c917 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes + reference: '' + 68239fd8-a74e-40ff-b5a5-7767bcd67ff2: !Template + answer_choices: Yes ||| No + id: 68239fd8-a74e-40ff-b5a5-7767bcd67ff2 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 6a940bc4-fb9e-4716-b7a0-9f0617c473f5: !Template + answer_choices: Yes ||| No + id: 6a940bc4-fb9e-4716-b7a0-9f0617c473f5 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes + reference: '' + 74826ac7-9337-4a80-9924-6d3b84565729: !Template answer_choices: A ||| B - id: 7aadbc5d-c690-4ed1-a85a-06634ee52a1c + id: 74826ac7-9337-4a80-9924-6d3b84565729 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer A or B. @@ -118,38 +165,22 @@ templates: original_task: true name: grammatical_which_one_A_B reference: '' - 83d3a865-30f4-443e-8368-7336e58295f9: !Template - answer_choices: Yes ||| No - id: 83d3a865-30f4-443e-8368-7336e58295f9 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no - reference: '' - 8d1b0578-195b-43c8-b4ae-690c36f580f0: !Template + 93166c80-802e-4c35-a340-f75b9ec7c80a: !Template answer_choices: A ||| B - id: 8d1b0578-195b-43c8-b4ae-690c36f580f0 + id: 93166c80-802e-4c35-a340-f75b9ec7c80a jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + Sentence A: {{ sentence_good }} Sentence B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} Sentence B: {{ sentence_good }} @@ -172,25 +203,9 @@ templates: original_task: true name: better_A_B reference: '' - b1e5958a-c528-4db3-a476-fb946b505800: !Template + b543de02-39cf-4086-bd21-b45ad43211ec: !Template answer_choices: Yes ||| No - id: b1e5958a-c528-4db3-a476-fb946b505800 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes - reference: '' - d3ef6e0b-bffe-4967-87b4-f62630976808: !Template - answer_choices: Yes ||| No - id: d3ef6e0b-bffe-4967-87b4-f62630976808 + id: b543de02-39cf-4086-bd21-b45ad43211ec jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -204,25 +219,9 @@ templates: original_task: false name: single_sentence_bad_yes_no reference: '' - d5649722-42bc-4fcb-be3d-3962366290c0: !Template - answer_choices: Yes ||| No - id: d5649722-42bc-4fcb-be3d-3962366290c0 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes - reference: '' - f16fb9fb-eb34-4f3c-b5a0-4c90a94fd92e: !Template + be72e247-5f10-4be1-a938-54bd469f443a: !Template answer_choices: 1 ||| 2 - id: f16fb9fb-eb34-4f3c-b5a0-4c90a94fd92e + id: be72e247-5f10-4be1-a938-54bd469f443a jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer 1 or 2. @@ -259,29 +258,30 @@ templates: original_task: true name: grammatical_which_one_1_2 reference: '' - fc54234e-a248-4bfc-b15c-f179bd434fa1: !Template + c8a0d6a0-0245-4471-b7bd-83d9159749f8: !Template answer_choices: A ||| B - id: fc54234e-a248-4bfc-b15c-f179bd434fa1 + id: c8a0d6a0-0245-4471-b7bd-83d9159749f8 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -295,5 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: grammatical_between_A_B reference: '' diff --git a/promptsource/templates/blimp/intransitive/templates.yaml b/promptsource/templates/blimp/intransitive/templates.yaml index 3bacdf798..57a6795f1 100644 --- a/promptsource/templates/blimp/intransitive/templates.yaml +++ b/promptsource/templates/blimp/intransitive/templates.yaml @@ -1,28 +1,28 @@ dataset: blimp subset: intransitive templates: - 0fe12db8-4370-423e-828a-def7a19983d5: !Template + 05fe66d3-f4bf-4987-bbdd-84e087faef22: !Template answer_choices: A ||| B - id: 0fe12db8-4370-423e-828a-def7a19983d5 + id: 05fe66d3-f4bf-4987-bbdd-84e087faef22 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + Sentence A: {{ sentence_good }} Sentence B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} Sentence B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order + Which one is a better sentence of English, B or A? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -37,58 +37,42 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B - reference: '' - 2f25caac-5beb-444f-aa9b-3d1138e2a5bc: !Template - answer_choices: Yes ||| No - id: 2f25caac-5beb-444f-aa9b-3d1138e2a5bc - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no + name: better_B_A reference: '' - 4ee3c279-2bd8-45b3-8a62-d09e9566c5a1: !Template + 135732f1-5ae4-4562-8bd2-c79e326c61f4: !Template answer_choices: Yes ||| No - id: 4ee3c279-2bd8-45b3-8a62-d09e9566c5a1 + id: 135732f1-5ae4-4562-8bd2-c79e326c61f4 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_good_no_yes + name: single_sentence_bad_yes_no reference: '' - 6643bb58-477f-4a75-9673-2617dd2e3772: !Template - answer_choices: 1 ||| 2 - id: 6643bb58-477f-4a75-9673-2617dd2e3772 + 1eefc31e-fc87-4284-8a2c-5c6d5c056099: !Template + answer_choices: A ||| B + id: 1eefc31e-fc87-4284-8a2c-5c6d5c056099 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -108,14 +92,30 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: grammatical_which_one_A_B + reference: '' + 3512c0b1-cbec-4c98-9b6b-93632d24ff5a: !Template + answer_choices: Yes ||| No + id: 3512c0b1-cbec-4c98-9b6b-93632d24ff5a + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes reference: '' - 6d0fb0f0-07fe-4786-b596-7602ea9e382d: !Template + 5b2f3a36-4482-4000-8a18-983177b4b79b: !Template answer_choices: 1 ||| 2 - id: 6d0fb0f0-07fe-4786-b596-7602ea9e382d + id: 5b2f3a36-4482-4000-8a18-983177b4b79b jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -147,27 +147,11 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 - reference: '' - 879691af-a2ec-4476-a1c0-0f652093a587: !Template - answer_choices: Yes ||| No - id: 879691af-a2ec-4476-a1c0-0f652093a587 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no + name: grammatical_which_one_1_2 reference: '' - a9f837e3-4a75-48b5-8fcf-4f18cb3e329d: !Template + 69bc50bd-327b-471d-9f6f-9e4c9054b119: !Template answer_choices: A ||| B - id: a9f837e3-4a75-48b5-8fcf-4f18cb3e329d + id: 69bc50bd-327b-471d-9f6f-9e4c9054b119 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer A or B. @@ -204,46 +188,29 @@ templates: original_task: true name: grammatical_between_A_B reference: '' - ad3d2795-6613-4392-bc1d-13f774928042: !Template - answer_choices: Yes ||| No - id: ad3d2795-6613-4392-bc1d-13f774928042 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes - reference: '' - ae60cb40-3c68-491d-98fa-b92af9291d9d: !Template + 8c673b2a-62f1-40d5-b11c-4e90e1fdfa00: !Template answer_choices: A ||| B - id: ae60cb40-3c68-491d-98fa-b92af9291d9d + id: 8c673b2a-62f1-40d5-b11c-4e90e1fdfa00 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -257,31 +224,32 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: better_A_B reference: '' - c329889d-2046-4ab7-bae8-abd4bbc639fd: !Template - answer_choices: A ||| B - id: c329889d-2046-4ab7-bae8-abd4bbc639fd + a6a13106-9d4d-45b4-8d4b-cd6749bb9436: !Template + answer_choices: 1 ||| 2 + id: a6a13106-9d4d-45b4-8d4b-cd6749bb9436 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -295,5 +263,37 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: grammatical_between_1_2 + reference: '' + abf0bad1-be17-4f0c-bfdc-1983155e6e0c: !Template + answer_choices: Yes ||| No + id: abf0bad1-be17-4f0c-bfdc-1983155e6e0c + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + efb1827e-a6d7-48bf-82cf-1550f11a48c3: !Template + answer_choices: Yes ||| No + id: efb1827e-a6d7-48bf-82cf-1550f11a48c3 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes reference: '' diff --git a/promptsource/templates/blimp/irregular_past_participle_adjectives/templates.yaml b/promptsource/templates/blimp/irregular_past_participle_adjectives/templates.yaml index 439abf7d3..05aa04fab 100644 --- a/promptsource/templates/blimp/irregular_past_participle_adjectives/templates.yaml +++ b/promptsource/templates/blimp/irregular_past_participle_adjectives/templates.yaml @@ -1,29 +1,30 @@ dataset: blimp subset: irregular_past_participle_adjectives templates: - 23bc54ae-098c-4bf1-ad60-a6cc489395f4: !Template + 01e46cac-569b-43e6-9bc0-569fc9f00fbd: !Template answer_choices: A ||| B - id: 23bc54ae-098c-4bf1-ad60-a6cc489395f4 + id: 01e46cac-569b-43e6-9bc0-569fc9f00fbd jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -37,11 +38,50 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: grammatical_between_A_B + reference: '' + 654feffd-c577-4f2c-a54c-4e17cc33a3a2: !Template + answer_choices: 1 ||| 2 + id: 654feffd-c577-4f2c-a54c-4e17cc33a3a2 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_1_2 reference: '' - 26a47924-2a4c-4c1e-92f6-22e78274bcc9: !Template + 97bb0e08-0820-4b56-81a9-c206264f4197: !Template answer_choices: A ||| B - id: 26a47924-2a4c-4c1e-92f6-22e78274bcc9 + id: 97bb0e08-0820-4b56-81a9-c206264f4197 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer A or B. @@ -78,13 +118,13 @@ templates: original_task: true name: grammatical_which_one_A_B reference: '' - 57f5a493-b0e2-4846-869a-5a69e5e7455e: !Template + b3328790-7d90-4dfb-9cf0-b2ff946346e1: !Template answer_choices: Yes ||| No - id: 57f5a493-b0e2-4846-869a-5a69e5e7455e + id: b3328790-7d90-4dfb-9cf0-b2ff946346e1 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -92,46 +132,46 @@ templates: metrics: - Accuracy original_task: false - name: single_sentence_good_yes_no + name: single_sentence_good_no_yes reference: '' - 5935513b-f7cf-4acf-8fcb-4d12b3e0c8d0: !Template + b70d30d9-77dd-455f-b138-82eb05beff7d: !Template answer_choices: Yes ||| No - id: 5935513b-f7cf-4acf-8fcb-4d12b3e0c8d0 + id: b70d30d9-77dd-455f-b138-82eb05beff7d jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_bad_no_yes + name: single_sentence_good_yes_no reference: '' - 65c4f840-4084-4526-af02-2f26edb24172: !Template + b8ca2315-d995-469c-a25d-5bcf20ed84a4: !Template answer_choices: A ||| B - id: 65c4f840-4084-4526-af02-2f26edb24172 + id: b8ca2315-d995-469c-a25d-5bcf20ed84a4 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + Sentence A: {{ sentence_good }} Sentence B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} Sentence B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order + Which one is a better sentence of English, A or B? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -146,66 +186,27 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A - reference: '' - 6fc75db1-2999-4a0d-964b-3fc1d65f7e55: !Template - answer_choices: 1 ||| 2 - id: 6fc75db1-2999-4a0d-964b-3fc1d65f7e55 - jinja: '{% set shuffled_order = [0, 1] | random %} - - Which one of the following sentences is grammatical? Please answer 1 or 2. - - {% if shuffled_order == 0 %} - - 1: {{ sentence_good }} - - 2: {{ sentence_bad }} - - {% else %} - - 1: {{ sentence_bad }} - - 2: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} - - {% endif %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: grammatical_which_one_1_2 + name: better_A_B reference: '' - 834fd501-fb19-449d-8f44-65ae2423904b: !Template + c6c7352a-5b68-4ca8-9db2-70902434b97e: !Template answer_choices: Yes ||| No - id: 834fd501-fb19-449d-8f44-65ae2423904b + id: c6c7352a-5b68-4ca8-9db2-70902434b97e jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_good_no_yes + name: single_sentence_bad_no_yes reference: '' - 8fcbb887-6c8d-4d85-a7ef-839db220d0c0: !Template + d59d4523-ea2b-4780-8a76-4ddf27f69ca6: !Template answer_choices: Yes ||| No - id: 8fcbb887-6c8d-4d85-a7ef-839db220d0c0 + id: d59d4523-ea2b-4780-8a76-4ddf27f69ca6 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -219,24 +220,24 @@ templates: original_task: false name: single_sentence_bad_yes_no reference: '' - b2ed7261-81f0-48ec-8f1e-b8fa01cfff6e: !Template - answer_choices: A ||| B - id: b2ed7261-81f0-48ec-8f1e-b8fa01cfff6e + ef8fe88c-c95f-4b97-9d09-0184c43f4244: !Template + answer_choices: 1 ||| 2 + id: ef8fe88c-c95f-4b97-9d09-0184c43f4244 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -256,32 +257,31 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: grammatical_which_one_1_2 reference: '' - edf23807-c574-4925-92cd-b2799a08bc4e: !Template - answer_choices: 1 ||| 2 - id: edf23807-c574-4925-92cd-b2799a08bc4e + ff43ecaf-279d-41a6-8519-d0c57b5d61ce: !Template + answer_choices: A ||| B + id: ff43ecaf-279d-41a6-8519-d0c57b5d61ce jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -295,5 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: better_B_A reference: '' diff --git a/promptsource/templates/blimp/irregular_past_participle_verbs/templates.yaml b/promptsource/templates/blimp/irregular_past_participle_verbs/templates.yaml index 88f2db717..79a56bb20 100644 --- a/promptsource/templates/blimp/irregular_past_participle_verbs/templates.yaml +++ b/promptsource/templates/blimp/irregular_past_participle_verbs/templates.yaml @@ -1,22 +1,22 @@ dataset: blimp subset: irregular_past_participle_verbs templates: - 11db3871-f573-4089-b0d2-0b8fff3b0b8c: !Template + 05aea489-316f-42e0-9868-e6e516fea9b0: !Template answer_choices: A ||| B - id: 11db3871-f573-4089-b0d2-0b8fff3b0b8c + id: 05aea489-316f-42e0-9868-e6e516fea9b0 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + Sentence A: {{ sentence_good }} Sentence B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} Sentence B: {{ sentence_good }} @@ -39,9 +39,9 @@ templates: original_task: true name: better_A_B reference: '' - 23f44d7d-cdd9-4194-8acc-b65205aed9b9: !Template + 2d2b924b-b765-41f6-8d6d-e8f6238d1abf: !Template answer_choices: A ||| B - id: 23f44d7d-cdd9-4194-8acc-b65205aed9b9 + id: 2d2b924b-b765-41f6-8d6d-e8f6238d1abf jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer A or B. @@ -78,9 +78,9 @@ templates: original_task: true name: grammatical_which_one_A_B reference: '' - 2ea2f62f-6940-406e-9f0d-126b714c4d5b: !Template + 464409e0-a6b0-4b72-b39b-e4d19557bec5: !Template answer_choices: A ||| B - id: 2ea2f62f-6940-406e-9f0d-126b714c4d5b + id: 464409e0-a6b0-4b72-b39b-e4d19557bec5 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer A or B. @@ -117,9 +117,48 @@ templates: original_task: true name: grammatical_between_A_B reference: '' - 3c3f23a1-ce5c-4219-befe-f613a054ed13: !Template + 6b8e17de-cd52-4744-aa19-05502547c983: !Template + answer_choices: 1 ||| 2 + id: 6b8e17de-cd52-4744-aa19-05502547c983 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_1_2 + reference: '' + 719d40ac-f8b6-4b71-a31b-600e493f6ee9: !Template answer_choices: Yes ||| No - id: 3c3f23a1-ce5c-4219-befe-f613a054ed13 + id: 719d40ac-f8b6-4b71-a31b-600e493f6ee9 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -133,22 +172,22 @@ templates: original_task: false name: single_sentence_good_yes_no reference: '' - 41bb15eb-77af-402b-9f30-e23f5eb39060: !Template + 73c02ed4-3d51-4394-9276-793d54d870a4: !Template answer_choices: A ||| B - id: 41bb15eb-77af-402b-9f30-e23f5eb39060 + id: 73c02ed4-3d51-4394-9276-793d54d870a4 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + Sentence A: {{ sentence_good }} Sentence B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} Sentence B: {{ sentence_good }} @@ -171,13 +210,13 @@ templates: original_task: true name: better_B_A reference: '' - 5b0ffc77-aac1-4aba-a4ca-d6217c02f690: !Template + 7e364fac-a76d-47d9-b660-a235be6ad22a: !Template answer_choices: Yes ||| No - id: 5b0ffc77-aac1-4aba-a4ca-d6217c02f690 + id: 7e364fac-a76d-47d9-b660-a235be6ad22a jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata @@ -185,14 +224,14 @@ templates: metrics: - Accuracy original_task: false - name: single_sentence_bad_yes_no + name: single_sentence_bad_no_yes reference: '' - 7917ccfe-7790-4969-bc66-0e2f30690088: !Template + 8342400f-5e9d-4867-bfae-59edecab513f: !Template answer_choices: 1 ||| 2 - id: 7917ccfe-7790-4969-bc66-0e2f30690088 + id: 8342400f-5e9d-4867-bfae-59edecab513f jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -224,76 +263,37 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: grammatical_between_1_2 reference: '' - b0a3745d-c47d-44c8-b730-c1cb6ee527ac: !Template + 94c81ab8-a021-4594-9e67-71b9a35154ec: !Template answer_choices: Yes ||| No - id: b0a3745d-c47d-44c8-b730-c1cb6ee527ac + id: 94c81ab8-a021-4594-9e67-71b9a35154ec jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_bad_no_yes + name: single_sentence_good_no_yes reference: '' - cbb18fd4-d194-4f2c-b34b-99c8a2b7b77c: !Template + d33b96d4-10ec-4778-aedc-e37d911cfd7f: !Template answer_choices: Yes ||| No - id: cbb18fd4-d194-4f2c-b34b-99c8a2b7b77c + id: d33b96d4-10ec-4778-aedc-e37d911cfd7f jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_good_no_yes - reference: '' - fbe74e44-6434-4389-94de-5360a1fde011: !Template - answer_choices: 1 ||| 2 - id: fbe74e44-6434-4389-94de-5360a1fde011 - jinja: '{% set shuffled_order = [0, 1] | random %} - - Between the two sentences, which one is grammatical? Please answer 1 or 2. - - {% if shuffled_order == 0 %} - - 1: {{ sentence_good }} - - 2: {{ sentence_bad }} - - {% else %} - - 1: {{ sentence_bad }} - - 2: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} - - {% endif %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: grammatical_between_1_2 + name: single_sentence_bad_yes_no reference: '' diff --git a/promptsource/templates/blimp/irregular_plural_subject_verb_agreement_1/templates.yaml b/promptsource/templates/blimp/irregular_plural_subject_verb_agreement_1/templates.yaml index 702c01c9d..2955fb531 100644 --- a/promptsource/templates/blimp/irregular_plural_subject_verb_agreement_1/templates.yaml +++ b/promptsource/templates/blimp/irregular_plural_subject_verb_agreement_1/templates.yaml @@ -1,22 +1,22 @@ dataset: blimp subset: irregular_plural_subject_verb_agreement_1 templates: - 24a53c55-2980-4599-b329-5ab988dd7b2a: !Template + 0b237ec0-4979-4207-b2da-b2ea4a51df1e: !Template answer_choices: A ||| B - id: 24a53c55-2980-4599-b329-5ab988dd7b2a + id: 0b237ec0-4979-4207-b2da-b2ea4a51df1e jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + Sentence A: {{ sentence_good }} Sentence B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} Sentence B: {{ sentence_good }} @@ -39,24 +39,24 @@ templates: original_task: true name: better_A_B reference: '' - 3a947519-9297-4256-8aee-075be3060c3e: !Template - answer_choices: 1 ||| 2 - id: 3a947519-9297-4256-8aee-075be3060c3e + 51be22a7-cfd5-47f0-84b2-fe4c4cf73adb: !Template + answer_choices: A ||| B + id: 51be22a7-cfd5-47f0-84b2-fe4c4cf73adb jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -76,26 +76,26 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: grammatical_between_A_B reference: '' - 438261f2-b8d6-4554-8b37-23427a9b5a4e: !Template - answer_choices: A ||| B - id: 438261f2-b8d6-4554-8b37-23427a9b5a4e + 632d0ed3-9a42-4b09-9895-e92e5abf51fa: !Template + answer_choices: 1 ||| 2 + id: 632d0ed3-9a42-4b09-9895-e92e5abf51fa jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -115,48 +115,31 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B - reference: '' - 91735f88-fd6b-45ed-8730-1dc85fcc1fb3: !Template - answer_choices: Yes ||| No - id: 91735f88-fd6b-45ed-8730-1dc85fcc1fb3 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no + name: grammatical_between_1_2 reference: '' - 960258d2-0bbe-4f6c-a7ca-85fa7a5efbfd: !Template + 666b1056-86bc-4eac-b193-c47308692e0c: !Template answer_choices: A ||| B - id: 960258d2-0bbe-4f6c-a7ca-85fa7a5efbfd + id: 666b1056-86bc-4eac-b193-c47308692e0c jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -170,11 +153,11 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: better_B_A reference: '' - c60e7e92-1538-4638-bf8a-5c70739bea92: !Template + 6be39d40-ddc8-4f8d-8b38-1439d2a0c897: !Template answer_choices: Yes ||| No - id: c60e7e92-1538-4638-bf8a-5c70739bea92 + id: 6be39d40-ddc8-4f8d-8b38-1439d2a0c897 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -188,9 +171,9 @@ templates: original_task: false name: single_sentence_good_no_yes reference: '' - de803bcc-9767-47cc-a176-a175ddc9fce6: !Template + 83c0cac5-5ee1-4fa2-a7f9-6b2e450de34c: !Template answer_choices: Yes ||| No - id: de803bcc-9767-47cc-a176-a175ddc9fce6 + id: 83c0cac5-5ee1-4fa2-a7f9-6b2e450de34c jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -204,12 +187,12 @@ templates: original_task: false name: single_sentence_bad_no_yes reference: '' - e51d328a-cd40-4be0-af2a-5c50b32ee303: !Template + 8e84f8d8-a18c-44a7-bc14-78b9fdc3e6ca: !Template answer_choices: 1 ||| 2 - id: e51d328a-cd40-4be0-af2a-5c50b32ee303 + id: 8e84f8d8-a18c-44a7-bc14-78b9fdc3e6ca jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -241,31 +224,48 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: grammatical_which_one_1_2 reference: '' - ecc18041-dd7e-44a2-8a1c-a376faeab14c: !Template + 9e3eb092-c65b-452e-989d-b6b9a1dd0617: !Template + answer_choices: Yes ||| No + id: 9e3eb092-c65b-452e-989d-b6b9a1dd0617 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + b8b3dd57-20b7-4d5e-9b16-64cca88f5502: !Template answer_choices: A ||| B - id: ecc18041-dd7e-44a2-8a1c-a376faeab14c + id: b8b3dd57-20b7-4d5e-9b16-64cca88f5502 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -279,11 +279,11 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: grammatical_which_one_A_B reference: '' - eef40269-9104-453c-a521-27dae1b187d9: !Template + ec92dd7e-d41f-456e-b3d7-78801bd84c54: !Template answer_choices: Yes ||| No - id: eef40269-9104-453c-a521-27dae1b187d9 + id: ec92dd7e-d41f-456e-b3d7-78801bd84c54 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" diff --git a/promptsource/templates/blimp/irregular_plural_subject_verb_agreement_2/templates.yaml b/promptsource/templates/blimp/irregular_plural_subject_verb_agreement_2/templates.yaml index 790b8615a..930746773 100644 --- a/promptsource/templates/blimp/irregular_plural_subject_verb_agreement_2/templates.yaml +++ b/promptsource/templates/blimp/irregular_plural_subject_verb_agreement_2/templates.yaml @@ -1,30 +1,29 @@ dataset: blimp subset: irregular_plural_subject_verb_agreement_2 templates: - 1f60b3c6-01fa-45e9-bb04-d8d57a9e770e: !Template - answer_choices: 1 ||| 2 - id: 1f60b3c6-01fa-45e9-bb04-d8d57a9e770e + 1131584f-b755-4041-8632-edd6be1d49ec: !Template + answer_choices: A ||| B + id: 1131584f-b755-4041-8632-edd6be1d49ec jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -38,32 +37,47 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: better_A_B reference: '' - 43af7924-0682-4f49-8fc6-acc3620a62f0: !Template - answer_choices: 1 ||| 2 - id: 43af7924-0682-4f49-8fc6-acc3620a62f0 + 17692107-00aa-4a55-83f2-0385580c64d5: !Template + answer_choices: Yes ||| No + id: 17692107-00aa-4a55-83f2-0385580c64d5 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 2df1de79-e0b7-41d8-a6e8-ff46a3b09769: !Template + answer_choices: A ||| B + id: 2df1de79-e0b7-41d8-a6e8-ff46a3b09769 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -77,15 +91,15 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: better_B_A reference: '' - 7f94f3a2-6c03-420b-a363-75cd2c95c05f: !Template + 39bb2c89-ec48-49ca-8cf2-0a4b9811f177: !Template answer_choices: Yes ||| No - id: 7f94f3a2-6c03-420b-a363-75cd2c95c05f + id: 39bb2c89-ec48-49ca-8cf2-0a4b9811f177 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -93,26 +107,42 @@ templates: metrics: - Accuracy original_task: false - name: single_sentence_good_yes_no + name: single_sentence_good_no_yes reference: '' - 8bec548b-cbd6-462d-92bb-6034bf57bc65: !Template - answer_choices: A ||| B - id: 8bec548b-cbd6-462d-92bb-6034bf57bc65 + 5828fcad-f8b2-4975-ac98-b363913a6cf0: !Template + answer_choices: Yes ||| No + id: 5828fcad-f8b2-4975-ac98-b363913a6cf0 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes + reference: '' + 770aa1ee-31cf-405e-b2f3-568b651d8ea1: !Template + answer_choices: 1 ||| 2 + id: 770aa1ee-31cf-405e-b2f3-568b651d8ea1 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -132,26 +162,26 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: grammatical_which_one_1_2 reference: '' - a274f8b4-f55f-436d-bb37-f0ed93a5eb17: !Template - answer_choices: A ||| B - id: a274f8b4-f55f-436d-bb37-f0ed93a5eb17 + 85283761-2238-492e-ab39-db0992b88cbc: !Template + answer_choices: 1 ||| 2 + id: 85283761-2238-492e-ab39-db0992b88cbc jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -171,63 +201,32 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B - reference: '' - a96f8259-6df9-4d65-b8a1-53894a99c0c6: !Template - answer_choices: Yes ||| No - id: a96f8259-6df9-4d65-b8a1-53894a99c0c6 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no - reference: '' - bed5216d-07b3-4f08-8927-c1008929a95a: !Template - answer_choices: Yes ||| No - id: bed5216d-07b3-4f08-8927-c1008929a95a - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes + name: grammatical_between_1_2 reference: '' - c75dd4fd-efed-4b14-8987-2fd4f429c1b2: !Template + 9cff0cd3-9611-4169-b3d6-3ea707dbb57c: !Template answer_choices: A ||| B - id: c75dd4fd-efed-4b14-8987-2fd4f429c1b2 + id: 9cff0cd3-9611-4169-b3d6-3ea707dbb57c jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -241,47 +240,48 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: grammatical_which_one_A_B reference: '' - d88168a9-462e-455f-b5c5-cbe5dd22b449: !Template + a51daee4-1098-4141-85fc-4887e8eb506f: !Template answer_choices: Yes ||| No - id: d88168a9-462e-455f-b5c5-cbe5dd22b449 + id: a51daee4-1098-4141-85fc-4887e8eb506f jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_bad_no_yes + name: single_sentence_good_yes_no reference: '' - f47179e9-df3e-4890-b722-2d8438724c49: !Template + e161cd15-e148-4a02-8a31-2ec90546fb05: !Template answer_choices: A ||| B - id: f47179e9-df3e-4890-b722-2d8438724c49 + id: e161cd15-e148-4a02-8a31-2ec90546fb05 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -295,5 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: grammatical_between_A_B reference: '' diff --git a/promptsource/templates/blimp/left_branch_island_echo_question/templates.yaml b/promptsource/templates/blimp/left_branch_island_echo_question/templates.yaml index 566b79789..c3fa4a49d 100644 --- a/promptsource/templates/blimp/left_branch_island_echo_question/templates.yaml +++ b/promptsource/templates/blimp/left_branch_island_echo_question/templates.yaml @@ -1,9 +1,9 @@ dataset: blimp subset: left_branch_island_echo_question templates: - 7523e51d-248c-4965-a94a-e7155bf752df: !Template + 00139aeb-52d3-47c1-a2fe-6b1e94f37f01: !Template answer_choices: A ||| B - id: 7523e51d-248c-4965-a94a-e7155bf752df + id: 00139aeb-52d3-47c1-a2fe-6b1e94f37f01 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer A or B. @@ -40,9 +40,9 @@ templates: original_task: true name: grammatical_between_A_B reference: '' - 79613311-d2b3-41b7-aaf3-f3e5a99fd63c: !Template + 17dc5b32-9315-430d-9958-9da439352732: !Template answer_choices: 1 ||| 2 - id: 79613311-d2b3-41b7-aaf3-f3e5a99fd63c + id: 17dc5b32-9315-430d-9958-9da439352732 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer 1 or 2. @@ -79,25 +79,9 @@ templates: original_task: true name: grammatical_which_one_1_2 reference: '' - 9b60e503-bf0c-4d5a-949b-ae81af206854: !Template - answer_choices: Yes ||| No - id: 9b60e503-bf0c-4d5a-949b-ae81af206854 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no - reference: '' - a7daf9c4-c4a3-4650-af89-e63370c96ac8: !Template + 399524ee-c0c0-436d-994b-4cda3d7586c5: !Template answer_choices: A ||| B - id: a7daf9c4-c4a3-4650-af89-e63370c96ac8 + id: 399524ee-c0c0-436d-994b-4cda3d7586c5 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer A or B. @@ -134,38 +118,22 @@ templates: original_task: true name: grammatical_which_one_A_B reference: '' - abcf99e4-c6b7-4408-be5b-7c35e84b6b41: !Template - answer_choices: Yes ||| No - id: abcf99e4-c6b7-4408-be5b-7c35e84b6b41 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes - reference: '' - c4e5cd99-6d94-436d-b171-68652a131c3e: !Template + 44ffe4cd-2787-4482-bc3f-a134a0d7db6c: !Template answer_choices: A ||| B - id: c4e5cd99-6d94-436d-b171-68652a131c3e + id: 44ffe4cd-2787-4482-bc3f-a134a0d7db6c jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + Sentence A: {{ sentence_good }} Sentence B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} Sentence B: {{ sentence_good }} @@ -188,9 +156,57 @@ templates: original_task: true name: better_B_A reference: '' - db819087-a96c-487d-aec7-2e623a22c3f4: !Template + 4b89b698-6c79-4eb0-9d19-2cac69b4cf3f: !Template + answer_choices: Yes ||| No + id: 4b89b698-6c79-4eb0-9d19-2cac69b4cf3f + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes + reference: '' + 5fb3ddb1-515b-45b7-b2bf-b6c3cee673e0: !Template + answer_choices: Yes ||| No + id: 5fb3ddb1-515b-45b7-b2bf-b6c3cee673e0 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes + reference: '' + ddea312a-bbf9-49d8-808a-834462fb44b9: !Template + answer_choices: Yes ||| No + id: ddea312a-bbf9-49d8-808a-834462fb44b9 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + e155bb05-1ef7-4f23-a1f3-3241fb177b0e: !Template answer_choices: Yes ||| No - id: db819087-a96c-487d-aec7-2e623a22c3f4 + id: e155bb05-1ef7-4f23-a1f3-3241fb177b0e jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -204,29 +220,30 @@ templates: original_task: false name: single_sentence_good_yes_no reference: '' - e27def85-1590-4c28-ad69-9a7d89c83b54: !Template - answer_choices: A ||| B - id: e27def85-1590-4c28-ad69-9a7d89c83b54 + f1b75254-853d-4c0b-9de2-f6363138639c: !Template + answer_choices: 1 ||| 2 + id: f1b75254-853d-4c0b-9de2-f6363138639c jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -240,48 +257,31 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B - reference: '' - e55d47cb-f5a0-494f-8f2d-d558e4dd3cca: !Template - answer_choices: Yes ||| No - id: e55d47cb-f5a0-494f-8f2d-d558e4dd3cca - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes + name: grammatical_between_1_2 reference: '' - fab92cfb-355c-4c52-b0a4-b2d68c47cf84: !Template - answer_choices: 1 ||| 2 - id: fab92cfb-355c-4c52-b0a4-b2d68c47cf84 + fa56226a-16f3-483c-b8bb-864d884c67aa: !Template + answer_choices: A ||| B + id: fa56226a-16f3-483c-b8bb-864d884c67aa jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -295,5 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: better_A_B reference: '' diff --git a/promptsource/templates/blimp/left_branch_island_simple_question/templates.yaml b/promptsource/templates/blimp/left_branch_island_simple_question/templates.yaml index 082474b5a..93a9321ca 100644 --- a/promptsource/templates/blimp/left_branch_island_simple_question/templates.yaml +++ b/promptsource/templates/blimp/left_branch_island_simple_question/templates.yaml @@ -1,61 +1,30 @@ dataset: blimp subset: left_branch_island_simple_question templates: - 015c5654-d491-4f48-be9e-4e5f5da0451c: !Template - answer_choices: Yes ||| No - id: 015c5654-d491-4f48-be9e-4e5f5da0451c - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes - reference: '' - 212f7bdb-78ef-4fa8-b418-527638cebdeb: !Template - answer_choices: Yes ||| No - id: 212f7bdb-78ef-4fa8-b418-527638cebdeb - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes - reference: '' - 2e2af07a-50b7-4579-9953-6d485d14acf4: !Template + 27a3f691-e5b1-49a3-8a2d-a5f8c63b9a21: !Template answer_choices: A ||| B - id: 2e2af07a-50b7-4579-9953-6d485d14acf4 + id: 27a3f691-e5b1-49a3-8a2d-a5f8c63b9a21 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -69,32 +38,31 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: grammatical_between_A_B reference: '' - 322706b1-4ccd-46c3-9cd1-a17cc4bc1b4c: !Template - answer_choices: 1 ||| 2 - id: 322706b1-4ccd-46c3-9cd1-a17cc4bc1b4c + 30ce7c9b-cbb3-4f4c-9298-086cbd601257: !Template + answer_choices: A ||| B + id: 30ce7c9b-cbb3-4f4c-9298-086cbd601257 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -108,14 +76,46 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: better_B_A + reference: '' + 33c54e67-6fad-4c75-ab25-0e4fa2737e56: !Template + answer_choices: Yes ||| No + id: 33c54e67-6fad-4c75-ab25-0e4fa2737e56 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes + reference: '' + 523c563f-7db2-4390-bd69-7d8f225bdd77: !Template + answer_choices: Yes ||| No + id: 523c563f-7db2-4390-bd69-7d8f225bdd77 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes reference: '' - 686f3007-e2b2-4d35-8d0d-3c4b8a61f300: !Template + 6d29c6bb-b345-4e0e-9b9c-9a7b05b5c2b3: !Template answer_choices: A ||| B - id: 686f3007-e2b2-4d35-8d0d-3c4b8a61f300 + id: 6d29c6bb-b345-4e0e-9b9c-9a7b05b5c2b3 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -147,11 +147,27 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: grammatical_which_one_A_B + reference: '' + 7fafffeb-e768-4abe-97e4-586e8043d477: !Template + answer_choices: Yes ||| No + id: 7fafffeb-e768-4abe-97e4-586e8043d477 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no reference: '' - 83dfb5ab-4462-4ce3-af1c-76e549ee03eb: !Template + 92c91987-ab83-45aa-bfb6-5ac7382e253c: !Template answer_choices: 1 ||| 2 - id: 83dfb5ab-4462-4ce3-af1c-76e549ee03eb + id: 92c91987-ab83-45aa-bfb6-5ac7382e253c jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer 1 or 2. @@ -188,46 +204,45 @@ templates: original_task: true name: grammatical_which_one_1_2 reference: '' - 99d6cb21-a620-42e4-9821-36a0e3c0eb1c: !Template + ad77d7b0-6cb3-4fe8-af1d-190707cb2ad2: !Template answer_choices: Yes ||| No - id: 99d6cb21-a620-42e4-9821-36a0e3c0eb1c + id: ad77d7b0-6cb3-4fe8-af1d-190707cb2ad2 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_good_yes_no + name: single_sentence_bad_yes_no reference: '' - d6493bf6-1d09-4b6c-b908-eea6ffc7d909: !Template + f6f176e5-b975-431c-a0fe-fca1ba07099f: !Template answer_choices: A ||| B - id: d6493bf6-1d09-4b6c-b908-eea6ffc7d909 + id: f6f176e5-b975-431c-a0fe-fca1ba07099f jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -241,31 +256,32 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: better_A_B reference: '' - ef134e9a-c50f-45a5-a49b-d56a408d1332: !Template - answer_choices: A ||| B - id: ef134e9a-c50f-45a5-a49b-d56a408d1332 + f945855c-7f51-4542-920f-855c44b11f96: !Template + answer_choices: 1 ||| 2 + id: f945855c-7f51-4542-920f-855c44b11f96 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -279,21 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A - reference: '' - f39202c2-c4af-4415-b9ff-5dbf2003a967: !Template - answer_choices: Yes ||| No - id: f39202c2-c4af-4415-b9ff-5dbf2003a967 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no + name: grammatical_between_1_2 reference: '' diff --git a/promptsource/templates/blimp/matrix_question_npi_licensor_present/templates.yaml b/promptsource/templates/blimp/matrix_question_npi_licensor_present/templates.yaml index 12d5ec4ef..174cd912d 100644 --- a/promptsource/templates/blimp/matrix_question_npi_licensor_present/templates.yaml +++ b/promptsource/templates/blimp/matrix_question_npi_licensor_present/templates.yaml @@ -1,45 +1,30 @@ dataset: blimp subset: matrix_question_npi_licensor_present templates: - 1bcb85f9-7794-45b2-a562-676bf5fa3afe: !Template - answer_choices: Yes ||| No - id: 1bcb85f9-7794-45b2-a562-676bf5fa3afe - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no - reference: '' - 252187ac-19ca-4c36-b201-0d15d0403972: !Template - answer_choices: A ||| B - id: 252187ac-19ca-4c36-b201-0d15d0403972 + 07b48aa1-d407-462a-b577-25aabf040692: !Template + answer_choices: 1 ||| 2 + id: 07b48aa1-d407-462a-b577-25aabf040692 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -53,32 +38,31 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: grammatical_which_one_1_2 reference: '' - 597931f7-e460-46de-847f-dac2d26a643e: !Template + 406d268d-3468-4965-a875-443b306b5615: !Template answer_choices: A ||| B - id: 597931f7-e460-46de-847f-dac2d26a643e + id: 406d268d-3468-4965-a875-443b306b5615 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -92,11 +76,11 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: better_B_A reference: '' - 6079f676-84cd-4683-98b9-1ac9e2d61980: !Template + 43e2da7e-7a7b-4021-84ad-814767601f3f: !Template answer_choices: A ||| B - id: 6079f676-84cd-4683-98b9-1ac9e2d61980 + id: 43e2da7e-7a7b-4021-84ad-814767601f3f jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer A or B. @@ -133,29 +117,62 @@ templates: original_task: true name: grammatical_which_one_A_B reference: '' - 7a7b4b49-9194-41d7-9c86-04ebae2ec3f7: !Template + 5e9a2cdb-70a4-4923-ad49-b2cd13c940c2: !Template + answer_choices: Yes ||| No + id: 5e9a2cdb-70a4-4923-ad49-b2cd13c940c2 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes + reference: '' + 6a3849a9-12db-42f6-8de6-79bc6cf648b5: !Template + answer_choices: Yes ||| No + id: 6a3849a9-12db-42f6-8de6-79bc6cf648b5 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 7d510aad-649c-4b6b-a18b-537a336fe4e1: !Template answer_choices: A ||| B - id: 7a7b4b49-9194-41d7-9c86-04ebae2ec3f7 + id: 7d510aad-649c-4b6b-a18b-537a336fe4e1 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -169,11 +186,27 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: grammatical_between_A_B reference: '' - 951c8b6d-e13b-41aa-bd13-04d2965491db: !Template + 83c9ae3f-92a4-4391-b7e1-a2cf107e7a01: !Template answer_choices: Yes ||| No - id: 951c8b6d-e13b-41aa-bd13-04d2965491db + id: 83c9ae3f-92a4-4391-b7e1-a2cf107e7a01 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 8fdbc6db-e6a7-4f7a-935e-64fd13f9f23d: !Template + answer_choices: Yes ||| No + id: 8fdbc6db-e6a7-4f7a-935e-64fd13f9f23d jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -187,9 +220,9 @@ templates: original_task: false name: single_sentence_good_no_yes reference: '' - a4079add-50bb-4530-a7d6-3807d6b64052: !Template + cf8ca906-3501-4036-8360-1c1e0f5da1a0: !Template answer_choices: 1 ||| 2 - id: a4079add-50bb-4530-a7d6-3807d6b64052 + id: cf8ca906-3501-4036-8360-1c1e0f5da1a0 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -226,46 +259,29 @@ templates: original_task: true name: grammatical_between_1_2 reference: '' - c1f1320d-8dd0-499b-97f4-004201f390f7: !Template - answer_choices: Yes ||| No - id: c1f1320d-8dd0-499b-97f4-004201f390f7 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes - reference: '' - c3bf87c9-06a7-4e51-abb0-6f51a9b62402: !Template - answer_choices: 1 ||| 2 - id: c3bf87c9-06a7-4e51-abb0-6f51a9b62402 + db3ba96d-9b12-48eb-ba18-61b8f57a94be: !Template + answer_choices: A ||| B + id: db3ba96d-9b12-48eb-ba18-61b8f57a94be jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -279,21 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 - reference: '' - f9fd9e89-7c1a-4509-bda6-fd5642715cda: !Template - answer_choices: Yes ||| No - id: f9fd9e89-7c1a-4509-bda6-fd5642715cda - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no + name: better_A_B reference: '' diff --git a/promptsource/templates/blimp/npi_present_1/templates.yaml b/promptsource/templates/blimp/npi_present_1/templates.yaml index c1697ab99..a0bb15987 100644 --- a/promptsource/templates/blimp/npi_present_1/templates.yaml +++ b/promptsource/templates/blimp/npi_present_1/templates.yaml @@ -1,9 +1,25 @@ dataset: blimp subset: npi_present_1 templates: - 32056d72-aea1-4e8d-ad7c-0f81b780cf37: !Template + 0ec6e8a2-6c7e-4029-899c-977d898dc720: !Template + answer_choices: Yes ||| No + id: 0ec6e8a2-6c7e-4029-899c-977d898dc720 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes + reference: '' + 295ddef9-8094-4241-9692-079048aa4cf8: !Template answer_choices: 1 ||| 2 - id: 32056d72-aea1-4e8d-ad7c-0f81b780cf37 + id: 295ddef9-8094-4241-9692-079048aa4cf8 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer 1 or 2. @@ -40,29 +56,46 @@ templates: original_task: true name: grammatical_which_one_1_2 reference: '' - 32f4aadc-ba57-4d97-bd43-985c7447ede6: !Template - answer_choices: A ||| B - id: 32f4aadc-ba57-4d97-bd43-985c7447ede6 + 3061fc20-3a06-4463-81dd-19eba967136e: !Template + answer_choices: Yes ||| No + id: 3061fc20-3a06-4463-81dd-19eba967136e + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 3625ff42-2148-45b9-9193-dfd98bb285cc: !Template + answer_choices: 1 ||| 2 + id: 3625ff42-2148-45b9-9193-dfd98bb285cc jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -76,40 +109,24 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A - reference: '' - 3736904c-b062-402d-acc1-8c946788c9dd: !Template - answer_choices: Yes ||| No - id: 3736904c-b062-402d-acc1-8c946788c9dd - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no + name: grammatical_between_1_2 reference: '' - 465acd78-1df0-4490-8c22-86f39387cd35: !Template + 3756c6cc-6f26-4f6a-b74c-4e5699ff39a2: !Template answer_choices: A ||| B - id: 465acd78-1df0-4490-8c22-86f39387cd35 + id: 3756c6cc-6f26-4f6a-b74c-4e5699ff39a2 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + Sentence A: {{ sentence_good }} Sentence B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} Sentence B: {{ sentence_good }} @@ -132,12 +149,12 @@ templates: original_task: true name: better_A_B reference: '' - 532862c3-b1ad-44a1-b819-2fbf4a3f3932: !Template + 55004e01-cbff-451d-b023-17f59cb9bbb2: !Template answer_choices: A ||| B - id: 532862c3-b1ad-44a1-b819-2fbf4a3f3932 + id: 55004e01-cbff-451d-b023-17f59cb9bbb2 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -169,30 +186,46 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: grammatical_between_A_B reference: '' - 76a64f9b-a4db-4099-afc0-4cdcc9c741b8: !Template + 5a855dac-928d-49bd-8e7e-843a27bbda2e: !Template answer_choices: Yes ||| No - id: 76a64f9b-a4db-4099-afc0-4cdcc9c741b8 + id: 5a855dac-928d-49bd-8e7e-843a27bbda2e jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_bad_yes_no + name: single_sentence_good_no_yes + reference: '' + 6326ea5b-a19c-468a-9abd-fb7f69ccb596: !Template + answer_choices: Yes ||| No + id: 6326ea5b-a19c-468a-9abd-fb7f69ccb596 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no reference: '' - 9eb6c3ab-5682-40d0-8576-462c27b33d95: !Template + 90f4ce45-a860-4124-aa2f-3bfd35acbd75: !Template answer_choices: A ||| B - id: 9eb6c3ab-5682-40d0-8576-462c27b33d95 + id: 90f4ce45-a860-4124-aa2f-3bfd35acbd75 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -224,64 +257,31 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B - reference: '' - a5d17a66-e645-4eaf-a2aa-ef8f7abc9dda: !Template - answer_choices: Yes ||| No - id: a5d17a66-e645-4eaf-a2aa-ef8f7abc9dda - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes - reference: '' - ca363a08-c111-42d6-b643-273f2ec56249: !Template - answer_choices: Yes ||| No - id: ca363a08-c111-42d6-b643-273f2ec56249 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes + name: grammatical_which_one_A_B reference: '' - cc274ad2-c68b-4cd5-bf5e-264b533ccd29: !Template - answer_choices: 1 ||| 2 - id: cc274ad2-c68b-4cd5-bf5e-264b533ccd29 + c0024b90-613a-4b4b-88bf-beeb05400208: !Template + answer_choices: A ||| B + id: c0024b90-613a-4b4b-88bf-beeb05400208 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -295,5 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: better_B_A reference: '' diff --git a/promptsource/templates/blimp/npi_present_2/templates.yaml b/promptsource/templates/blimp/npi_present_2/templates.yaml index d402d0af9..2432b0c7d 100644 --- a/promptsource/templates/blimp/npi_present_2/templates.yaml +++ b/promptsource/templates/blimp/npi_present_2/templates.yaml @@ -1,29 +1,46 @@ dataset: blimp subset: npi_present_2 templates: - 4d9be08d-1a50-4c43-a746-cfa4b4a40d63: !Template - answer_choices: A ||| B - id: 4d9be08d-1a50-4c43-a746-cfa4b4a40d63 + 0b89e24c-7d21-4955-ae69-f3c472a026b8: !Template + answer_choices: Yes ||| No + id: 0b89e24c-7d21-4955-ae69-f3c472a026b8 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes + reference: '' + 3993b6bc-0ac2-46c3-977e-fb57f9817386: !Template + answer_choices: 1 ||| 2 + id: 3993b6bc-0ac2-46c3-977e-fb57f9817386 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -37,42 +54,26 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A - reference: '' - 6f4974be-561f-4cf3-b324-2e4d0c0a6c02: !Template - answer_choices: Yes ||| No - id: 6f4974be-561f-4cf3-b324-2e4d0c0a6c02 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no + name: grammatical_which_one_1_2 reference: '' - 964a23ad-c863-4905-9d95-89f2f1e719aa: !Template - answer_choices: 1 ||| 2 - id: 964a23ad-c863-4905-9d95-89f2f1e719aa + 40251c2f-cd5b-4325-a7a0-37ad7a518376: !Template + answer_choices: A ||| B + id: 40251c2f-cd5b-4325-a7a0-37ad7a518376 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -92,42 +93,26 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 - reference: '' - a0de51d9-4521-488f-add6-8800014e8d2a: !Template - answer_choices: Yes ||| No - id: a0de51d9-4521-488f-add6-8800014e8d2a - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no + name: grammatical_which_one_A_B reference: '' - ad3bd372-17e6-49fc-acf1-0646752ef73d: !Template - answer_choices: 1 ||| 2 - id: ad3bd372-17e6-49fc-acf1-0646752ef73d + 62439fab-8be3-47d9-968c-23af62dfae28: !Template + answer_choices: A ||| B + id: 62439fab-8be3-47d9-968c-23af62dfae28 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -147,32 +132,31 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: grammatical_between_A_B reference: '' - b6a4d35f-9a39-477e-be91-767599992071: !Template + 913c63c8-c47a-4fca-b2b9-24536c2b2e55: !Template answer_choices: A ||| B - id: b6a4d35f-9a39-477e-be91-767599992071 + id: 913c63c8-c47a-4fca-b2b9-24536c2b2e55 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -186,46 +170,46 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: better_A_B reference: '' - c3d3c859-4637-45ff-93c2-a5e28786df26: !Template + 9616903f-3cc0-4a4b-91dc-1a34a0e05e94: !Template answer_choices: Yes ||| No - id: c3d3c859-4637-45ff-93c2-a5e28786df26 + id: 9616903f-3cc0-4a4b-91dc-1a34a0e05e94 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_good_no_yes + name: single_sentence_bad_yes_no reference: '' - dcae6658-26fe-47d8-bd56-fa4575aa2e82: !Template + aa523410-dc51-4aac-bfa9-e98263a39f31: !Template answer_choices: A ||| B - id: dcae6658-26fe-47d8-bd56-fa4575aa2e82 + id: aa523410-dc51-4aac-bfa9-e98263a39f31 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + Sentence A: {{ sentence_good }} Sentence B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} Sentence B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order + Which one is a better sentence of English, B or A? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -240,42 +224,58 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: better_B_A reference: '' - e7fed14a-3275-4240-b5f8-4db301eab377: !Template + accced8c-ab98-4944-b2b1-348933bdcf05: !Template answer_choices: Yes ||| No - id: e7fed14a-3275-4240-b5f8-4db301eab377 + id: accced8c-ab98-4944-b2b1-348933bdcf05 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_bad_no_yes + name: single_sentence_good_yes_no reference: '' - f304dead-d2a1-4000-9dc7-41eb52ce4834: !Template - answer_choices: A ||| B - id: f304dead-d2a1-4000-9dc7-41eb52ce4834 + f33d2c45-e5d6-450c-8252-e57c8638d921: !Template + answer_choices: Yes ||| No + id: f33d2c45-e5d6-450c-8252-e57c8638d921 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes + reference: '' + fcb15bb8-18ce-4a4d-80df-5c7d61d8a1e2: !Template + answer_choices: 1 ||| 2 + id: fcb15bb8-18ce-4a4d-80df-5c7d61d8a1e2 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -295,5 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: grammatical_between_1_2 reference: '' diff --git a/promptsource/templates/blimp/only_npi_licensor_present/templates.yaml b/promptsource/templates/blimp/only_npi_licensor_present/templates.yaml index 3b35e6478..6611931f5 100644 --- a/promptsource/templates/blimp/only_npi_licensor_present/templates.yaml +++ b/promptsource/templates/blimp/only_npi_licensor_present/templates.yaml @@ -1,30 +1,29 @@ dataset: blimp subset: only_npi_licensor_present templates: - 153a6b05-0890-403a-8570-4b991b48019f: !Template + 1b9ca899-4b67-4de0-8c43-4539a71fa4b8: !Template answer_choices: A ||| B - id: 153a6b05-0890-403a-8570-4b991b48019f + id: 1b9ca899-4b67-4de0-8c43-4539a71fa4b8 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -38,31 +37,64 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: better_B_A reference: '' - 2013eb76-f3d2-407a-a267-4c993c00732a: !Template - answer_choices: A ||| B - id: 2013eb76-f3d2-407a-a267-4c993c00732a + 37d70401-ea98-4863-99ac-ab652ce94678: !Template + answer_choices: Yes ||| No + id: 37d70401-ea98-4863-99ac-ab652ce94678 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes + reference: '' + 746cf635-d43d-453c-bfe6-c397cd971e93: !Template + answer_choices: Yes ||| No + id: 746cf635-d43d-453c-bfe6-c397cd971e93 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 95ea0068-8f46-4d77-a9a0-43e659ba5453: !Template + answer_choices: 1 ||| 2 + id: 95ea0068-8f46-4d77-a9a0-43e659ba5453 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -76,11 +108,27 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: grammatical_between_1_2 reference: '' - 524b2e9d-9f5c-41ea-aad4-85692ce4b7f6: !Template + abe1b796-d2d8-49b6-a86a-cb5e7325d699: !Template + answer_choices: Yes ||| No + id: abe1b796-d2d8-49b6-a86a-cb5e7325d699 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes + reference: '' + b6fef585-cb6c-4fd7-9b9d-4791d1a94951: !Template answer_choices: 1 ||| 2 - id: 524b2e9d-9f5c-41ea-aad4-85692ce4b7f6 + id: b6fef585-cb6c-4fd7-9b9d-4791d1a94951 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer 1 or 2. @@ -117,12 +165,12 @@ templates: original_task: true name: grammatical_which_one_1_2 reference: '' - 870c9a98-31a8-423f-a030-2b6968208898: !Template + ca2daa84-9dd4-4627-ad69-3b2df3cedd38: !Template answer_choices: A ||| B - id: 870c9a98-31a8-423f-a030-2b6968208898 + id: ca2daa84-9dd4-4627-ad69-3b2df3cedd38 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -154,15 +202,15 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: grammatical_between_A_B reference: '' - ac296786-c38f-4a14-967d-0f618ea7d3cc: !Template + cfceb31c-4da1-4317-88d5-65be913ee254: !Template answer_choices: Yes ||| No - id: ac296786-c38f-4a14-967d-0f618ea7d3cc + id: cfceb31c-4da1-4317-88d5-65be913ee254 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata @@ -170,63 +218,32 @@ templates: metrics: - Accuracy original_task: false - name: single_sentence_bad_no_yes - reference: '' - b705748a-13d5-4e43-b827-ecfa8926b4ca: !Template - answer_choices: Yes ||| No - id: b705748a-13d5-4e43-b827-ecfa8926b4ca - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no - reference: '' - f0c371af-1839-448d-9dce-89fdf7388449: !Template - answer_choices: Yes ||| No - id: f0c371af-1839-448d-9dce-89fdf7388449 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes + name: single_sentence_bad_yes_no reference: '' - f4e6761a-a597-4df7-b449-04ffb357e1c8: !Template + ec13eac1-3af6-4558-a70d-51a228d776b5: !Template answer_choices: A ||| B - id: f4e6761a-a597-4df7-b449-04ffb357e1c8 + id: ec13eac1-3af6-4558-a70d-51a228d776b5 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -240,48 +257,31 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A - reference: '' - f54d7542-0a50-49d1-ab19-d839ea705642: !Template - answer_choices: Yes ||| No - id: f54d7542-0a50-49d1-ab19-d839ea705642 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no + name: grammatical_which_one_A_B reference: '' - fb88f4f5-db5b-4af8-b38b-c6ffb5eaa1ad: !Template - answer_choices: 1 ||| 2 - id: fb88f4f5-db5b-4af8-b38b-c6ffb5eaa1ad + fc585a04-0b35-42cd-86bd-34cbdd074fd5: !Template + answer_choices: A ||| B + id: fc585a04-0b35-42cd-86bd-34cbdd074fd5 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -295,5 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: better_A_B reference: '' diff --git a/promptsource/templates/blimp/only_npi_scope/templates.yaml b/promptsource/templates/blimp/only_npi_scope/templates.yaml index 2435a6cb2..f7851ba01 100644 --- a/promptsource/templates/blimp/only_npi_scope/templates.yaml +++ b/promptsource/templates/blimp/only_npi_scope/templates.yaml @@ -1,29 +1,30 @@ dataset: blimp subset: only_npi_scope templates: - 05ee24e8-90c9-4dee-8f50-f5badda163ef: !Template - answer_choices: A ||| B - id: 05ee24e8-90c9-4dee-8f50-f5badda163ef + 3482c891-a0ba-4554-b669-64be4a25975d: !Template + answer_choices: 1 ||| 2 + id: 3482c891-a0ba-4554-b669-64be4a25975d jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -37,32 +38,47 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: grammatical_which_one_1_2 + reference: '' + 3ca02c3f-b858-4e4c-8260-21bf2c1a2b34: !Template + answer_choices: Yes ||| No + id: 3ca02c3f-b858-4e4c-8260-21bf2c1a2b34 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no reference: '' - 22fa63d8-54d8-47cf-b8b2-5bcdd837c3fe: !Template + 5e9be923-8cf8-41e9-b384-4ab54d473cd4: !Template answer_choices: A ||| B - id: 22fa63d8-54d8-47cf-b8b2-5bcdd837c3fe + id: 5e9be923-8cf8-41e9-b384-4ab54d473cd4 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -76,11 +92,11 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: better_A_B reference: '' - 2e1af49d-0122-4e6f-b63c-ff97ae661aca: !Template + af1d5046-01a1-4036-879d-cbf1952c3b03: !Template answer_choices: 1 ||| 2 - id: 2e1af49d-0122-4e6f-b63c-ff97ae661aca + id: af1d5046-01a1-4036-879d-cbf1952c3b03 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -117,24 +133,40 @@ templates: original_task: true name: grammatical_between_1_2 reference: '' - 33504196-78bc-4f05-917d-09ad26a455d2: !Template - answer_choices: 1 ||| 2 - id: 33504196-78bc-4f05-917d-09ad26a455d2 + c80b4a8c-c786-4c45-886f-37c7a26a67b7: !Template + answer_choices: Yes ||| No + id: c80b4a8c-c786-4c45-886f-37c7a26a67b7 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes + reference: '' + de665624-ec21-4256-b745-5c0380a10c31: !Template + answer_choices: A ||| B + id: de665624-ec21-4256-b745-5c0380a10c31 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -154,80 +186,31 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 - reference: '' - 5768a12d-ad5f-430c-aa56-e9e171647568: !Template - answer_choices: Yes ||| No - id: 5768a12d-ad5f-430c-aa56-e9e171647568 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no - reference: '' - 580641d7-f207-4abc-a8d9-22bd75be999a: !Template - answer_choices: Yes ||| No - id: 580641d7-f207-4abc-a8d9-22bd75be999a - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no - reference: '' - 75ac5fed-3d82-4903-a96d-05f330418b11: !Template - answer_choices: Yes ||| No - id: 75ac5fed-3d82-4903-a96d-05f330418b11 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes + name: grammatical_between_A_B reference: '' - e04d4f73-b27b-4ae2-802f-7a460939f37e: !Template + de7f9e83-effa-4aba-a3c5-695678c57b37: !Template answer_choices: A ||| B - id: e04d4f73-b27b-4ae2-802f-7a460939f37e + id: de7f9e83-effa-4aba-a3c5-695678c57b37 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -241,11 +224,11 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: better_B_A reference: '' - ea1796b8-c964-4040-886b-a1d10bf81f4e: !Template + e876bb66-ec5b-4fa8-be20-0ea7ff7ff92c: !Template answer_choices: Yes ||| No - id: ea1796b8-c964-4040-886b-a1d10bf81f4e + id: e876bb66-ec5b-4fa8-be20-0ea7ff7ff92c jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -259,29 +242,46 @@ templates: original_task: false name: single_sentence_bad_no_yes reference: '' - efbc668a-2116-4ad3-8cee-2cc11d7eb59e: !Template + f35d59da-a22a-444f-8762-c878586a054f: !Template + answer_choices: Yes ||| No + id: f35d59da-a22a-444f-8762-c878586a054f + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + f98213c5-0013-4edc-a0b3-7e0ac1b3738c: !Template answer_choices: A ||| B - id: efbc668a-2116-4ad3-8cee-2cc11d7eb59e + id: f98213c5-0013-4edc-a0b3-7e0ac1b3738c jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -295,5 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: grammatical_which_one_A_B reference: '' diff --git a/promptsource/templates/blimp/passive_1/templates.yaml b/promptsource/templates/blimp/passive_1/templates.yaml index 66040168c..9c5ce8a11 100644 --- a/promptsource/templates/blimp/passive_1/templates.yaml +++ b/promptsource/templates/blimp/passive_1/templates.yaml @@ -1,9 +1,9 @@ dataset: blimp subset: passive_1 templates: - 7775043b-3766-49b0-8666-6a3e6fb5c1d7: !Template + 0eafeb00-4c83-423a-8b6e-9cf3f50f6514: !Template answer_choices: A ||| B - id: 7775043b-3766-49b0-8666-6a3e6fb5c1d7 + id: 0eafeb00-4c83-423a-8b6e-9cf3f50f6514 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer A or B. @@ -40,29 +40,46 @@ templates: original_task: true name: grammatical_between_A_B reference: '' - 7d830b2b-2599-4770-9a59-ccf0138042e1: !Template - answer_choices: A ||| B - id: 7d830b2b-2599-4770-9a59-ccf0138042e1 + 289e788b-2c6a-4e27-bed5-787b3a301783: !Template + answer_choices: Yes ||| No + id: 289e788b-2c6a-4e27-bed5-787b3a301783 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes + reference: '' + 63202675-475d-4384-8309-6a1cec1a4436: !Template + answer_choices: 1 ||| 2 + id: 63202675-475d-4384-8309-6a1cec1a4436 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -76,56 +93,24 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A - reference: '' - 85461fa8-c978-4ed4-860a-c72ab86f3711: !Template - answer_choices: Yes ||| No - id: 85461fa8-c978-4ed4-860a-c72ab86f3711 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no - reference: '' - 90cd6e56-369d-4d07-93b0-dc53d2bd87fd: !Template - answer_choices: Yes ||| No - id: 90cd6e56-369d-4d07-93b0-dc53d2bd87fd - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes + name: grammatical_which_one_1_2 reference: '' - 9882eac8-a9c9-4302-bde6-f3beaa825fcf: !Template + 722af26b-dd76-48e5-a610-dd59fb06a22e: !Template answer_choices: A ||| B - id: 9882eac8-a9c9-4302-bde6-f3beaa825fcf + id: 722af26b-dd76-48e5-a610-dd59fb06a22e jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + Sentence A: {{ sentence_good }} Sentence B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} Sentence B: {{ sentence_good }} @@ -148,9 +133,9 @@ templates: original_task: true name: better_A_B reference: '' - 9caed5d3-2580-4506-91de-716d1fc35cff: !Template + 7809a036-4a53-48c7-9887-03e2366933b7: !Template answer_choices: 1 ||| 2 - id: 9caed5d3-2580-4506-91de-716d1fc35cff + id: 7809a036-4a53-48c7-9887-03e2366933b7 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -187,46 +172,29 @@ templates: original_task: true name: grammatical_between_1_2 reference: '' - b25be334-ab34-4074-b790-43247c086faf: !Template - answer_choices: Yes ||| No - id: b25be334-ab34-4074-b790-43247c086faf - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no - reference: '' - caedb3f8-c46a-4820-9f34-7ea7701e2777: !Template - answer_choices: 1 ||| 2 - id: caedb3f8-c46a-4820-9f34-7ea7701e2777 + b5d41404-3b11-4745-843e-527baeeb536e: !Template + answer_choices: A ||| B + id: b5d41404-3b11-4745-843e-527baeeb536e jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -240,11 +208,11 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: better_B_A reference: '' - e83d5379-0a3a-4f2a-90a5-ea433e1fe667: !Template + cfd42573-720e-4014-abba-3146aaa39ec1: !Template answer_choices: A ||| B - id: e83d5379-0a3a-4f2a-90a5-ea433e1fe667 + id: cfd42573-720e-4014-abba-3146aaa39ec1 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer A or B. @@ -281,9 +249,9 @@ templates: original_task: true name: grammatical_which_one_A_B reference: '' - f7a81173-2aee-460e-9205-85208d661bbd: !Template + e1014909-5b42-41ff-9ed0-2e432975de51: !Template answer_choices: Yes ||| No - id: f7a81173-2aee-460e-9205-85208d661bbd + id: e1014909-5b42-41ff-9ed0-2e432975de51 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -297,3 +265,35 @@ templates: original_task: false name: single_sentence_bad_no_yes reference: '' + ef7c1343-d328-4d10-a777-07cab171e674: !Template + answer_choices: Yes ||| No + id: ef7c1343-d328-4d10-a777-07cab171e674 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + efc34c01-f855-49c8-830d-a60f3422ccc4: !Template + answer_choices: Yes ||| No + id: efc34c01-f855-49c8-830d-a60f3422ccc4 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' diff --git a/promptsource/templates/blimp/passive_2/templates.yaml b/promptsource/templates/blimp/passive_2/templates.yaml index 31c23eb25..4785665a2 100644 --- a/promptsource/templates/blimp/passive_2/templates.yaml +++ b/promptsource/templates/blimp/passive_2/templates.yaml @@ -1,9 +1,9 @@ dataset: blimp subset: passive_2 templates: - 024782db-3e00-4f41-bea0-2ce72772245b: !Template + 32f79d36-e1b0-4ce5-affc-8129311b6e22: !Template answer_choices: A ||| B - id: 024782db-3e00-4f41-bea0-2ce72772245b + id: 32f79d36-e1b0-4ce5-affc-8129311b6e22 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer A or B. @@ -40,29 +40,13 @@ templates: original_task: true name: grammatical_between_A_B reference: '' - 0c2989e5-bd53-4e90-883d-065e752a4ca9: !Template + 361d1523-8535-450a-9162-db2d45344194: !Template answer_choices: Yes ||| No - id: 0c2989e5-bd53-4e90-883d-065e752a4ca9 + id: 361d1523-8535-450a-9162-db2d45344194 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes - reference: '' - 14a71122-0ed3-4f5d-be71-9317668e415e: !Template - answer_choices: Yes ||| No - id: 14a71122-0ed3-4f5d-be71-9317668e415e - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -70,27 +54,11 @@ templates: metrics: - Accuracy original_task: false - name: single_sentence_good_yes_no - reference: '' - 1e7bd836-30ce-4177-869b-67d097252829: !Template - answer_choices: Yes ||| No - id: 1e7bd836-30ce-4177-869b-67d097252829 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no + name: single_sentence_good_no_yes reference: '' - 4422bec7-971f-4e38-af19-354ba9a225d8: !Template + 731b4c9b-2c37-4b1f-8a32-000f3ebdcac3: !Template answer_choices: 1 ||| 2 - id: 4422bec7-971f-4e38-af19-354ba9a225d8 + id: 731b4c9b-2c37-4b1f-8a32-000f3ebdcac3 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -127,24 +95,24 @@ templates: original_task: true name: grammatical_between_1_2 reference: '' - 7ae6df52-8225-4af7-acec-920fc523d0f1: !Template - answer_choices: 1 ||| 2 - id: 7ae6df52-8225-4af7-acec-920fc523d0f1 + 7fd25e68-074f-498f-835a-44f1fac35a21: !Template + answer_choices: A ||| B + id: 7fd25e68-074f-498f-835a-44f1fac35a21 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -164,31 +132,64 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: grammatical_which_one_A_B reference: '' - 9e42bfca-33ae-4525-9c3a-6ee2f14e1884: !Template - answer_choices: A ||| B - id: 9e42bfca-33ae-4525-9c3a-6ee2f14e1884 + 96bf0a44-1f13-4fed-9fc0-3a817112cc67: !Template + answer_choices: Yes ||| No + id: 96bf0a44-1f13-4fed-9fc0-3a817112cc67 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes + reference: '' + 98b68892-3302-448a-8243-d2b178a3f522: !Template + answer_choices: Yes ||| No + id: 98b68892-3302-448a-8243-d2b178a3f522 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + be971081-1ba2-4a63-9c5a-9419e4555c93: !Template + answer_choices: 1 ||| 2 + id: be971081-1ba2-4a63-9c5a-9419e4555c93 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -202,46 +203,30 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A - reference: '' - a8f3e69d-0c1e-4828-bcb2-9bced3339bab: !Template - answer_choices: Yes ||| No - id: a8f3e69d-0c1e-4828-bcb2-9bced3339bab - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes + name: grammatical_which_one_1_2 reference: '' - b3933281-0de4-44d0-9975-68311a6ab796: !Template + c1fcde28-55da-41c7-8a76-58b0850aa762: !Template answer_choices: A ||| B - id: b3933281-0de4-44d0-9975-68311a6ab796 + id: c1fcde28-55da-41c7-8a76-58b0850aa762 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + Sentence A: {{ sentence_good }} Sentence B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} Sentence B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order + Which one is a better sentence of English, B or A? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -256,32 +241,31 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: better_B_A reference: '' - b942d542-d6cd-4105-8630-f6d5255855b8: !Template + d11adddf-ae9a-47e1-a7b1-bbc0ed7586c5: !Template answer_choices: A ||| B - id: b942d542-d6cd-4105-8630-f6d5255855b8 + id: d11adddf-ae9a-47e1-a7b1-bbc0ed7586c5 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -295,5 +279,21 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: better_A_B + reference: '' + dc1309cf-3082-4fcb-86a0-6bccb28b15af: !Template + answer_choices: Yes ||| No + id: dc1309cf-3082-4fcb-86a0-6bccb28b15af + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no reference: '' diff --git a/promptsource/templates/blimp/principle_A_c_command/templates.yaml b/promptsource/templates/blimp/principle_A_c_command/templates.yaml index 296b9c13c..ab77ebcb6 100644 --- a/promptsource/templates/blimp/principle_A_c_command/templates.yaml +++ b/promptsource/templates/blimp/principle_A_c_command/templates.yaml @@ -1,62 +1,29 @@ dataset: blimp subset: principle_A_c_command templates: - 28c50f2c-7001-403e-ba55-baab00d9c6b1: !Template - answer_choices: Yes ||| No - id: 28c50f2c-7001-403e-ba55-baab00d9c6b1 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes - reference: '' - 38f2a3d9-a0f0-4317-a59a-a8fb1a9823c7: !Template - answer_choices: Yes ||| No - id: 38f2a3d9-a0f0-4317-a59a-a8fb1a9823c7 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no - reference: '' - 3a9a578f-5e96-428d-8dea-9a8098291083: !Template - answer_choices: 1 ||| 2 - id: 3a9a578f-5e96-428d-8dea-9a8098291083 + 266eacf1-3d82-4adc-bcbd-24a0f45dbaef: !Template + answer_choices: A ||| B + id: 266eacf1-3d82-4adc-bcbd-24a0f45dbaef jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -70,30 +37,14 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 - reference: '' - 53b5d892-53ab-4205-a47b-88fc01444764: !Template - answer_choices: Yes ||| No - id: 53b5d892-53ab-4205-a47b-88fc01444764 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no + name: better_A_B reference: '' - 72be970a-653e-44c5-beef-e445239964fa: !Template + 528f6538-e203-431e-99c4-c99c13ec2741: !Template answer_choices: 1 ||| 2 - id: 72be970a-653e-44c5-beef-e445239964fa + id: 528f6538-e203-431e-99c4-c99c13ec2741 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -125,14 +76,14 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: grammatical_between_1_2 reference: '' - 9018b162-1d54-4a77-b5bf-552263b20447: !Template + 624031bc-9c70-4dfe-82d1-68ccb8ca0b58: !Template answer_choices: A ||| B - id: 9018b162-1d54-4a77-b5bf-552263b20447 + id: 624031bc-9c70-4dfe-82d1-68ccb8ca0b58 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -164,31 +115,48 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: grammatical_which_one_A_B + reference: '' + 6df7b6d2-e50d-4386-a836-ab1d627ff06c: !Template + answer_choices: Yes ||| No + id: 6df7b6d2-e50d-4386-a836-ab1d627ff06c + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes reference: '' - 95a67aa0-f823-4a92-a3ab-cf4cc4f724e3: !Template + 878859f7-80d6-49b2-b11f-750b6e945b7b: !Template answer_choices: A ||| B - id: 95a67aa0-f823-4a92-a3ab-cf4cc4f724e3 + id: 878859f7-80d6-49b2-b11f-750b6e945b7b jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -202,30 +170,46 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: grammatical_between_A_B + reference: '' + 8fcb6fdb-0783-4ec0-88e4-99d525ad09c1: !Template + answer_choices: Yes ||| No + id: 8fcb6fdb-0783-4ec0-88e4-99d525ad09c1 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no reference: '' - 96442dc1-11da-4acb-9c13-99239aac99d8: !Template + c18476f3-55fa-4381-8ebd-14d7ad7cd4e1: !Template answer_choices: A ||| B - id: 96442dc1-11da-4acb-9c13-99239aac99d8 + id: c18476f3-55fa-4381-8ebd-14d7ad7cd4e1 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + Sentence A: {{ sentence_good }} Sentence B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} Sentence B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order + Which one is a better sentence of English, B or A? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -240,26 +224,26 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: better_B_A reference: '' - ae57ebcd-d036-4486-9920-82c84e99702e: !Template - answer_choices: A ||| B - id: ae57ebcd-d036-4486-9920-82c84e99702e + d058331f-baab-4e84-b7ff-fdfc2afb6551: !Template + answer_choices: 1 ||| 2 + id: d058331f-baab-4e84-b7ff-fdfc2afb6551 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -279,21 +263,37 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: grammatical_which_one_1_2 reference: '' - f4decd0e-8077-49d0-95db-65ac8cbee793: !Template + d673932a-84b4-4e40-90d9-0d3a4b51a25a: !Template answer_choices: Yes ||| No - id: f4decd0e-8077-49d0-95db-65ac8cbee793 + id: d673932a-84b4-4e40-90d9-0d3a4b51a25a jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_good_no_yes + name: single_sentence_bad_yes_no + reference: '' + e1e35e99-b548-45f3-b550-58a16e93e127: !Template + answer_choices: Yes ||| No + id: e1e35e99-b548-45f3-b550-58a16e93e127 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes reference: '' diff --git a/promptsource/templates/blimp/principle_A_case_1/templates.yaml b/promptsource/templates/blimp/principle_A_case_1/templates.yaml index 50ae16dc3..d4f98c68c 100644 --- a/promptsource/templates/blimp/principle_A_case_1/templates.yaml +++ b/promptsource/templates/blimp/principle_A_case_1/templates.yaml @@ -1,30 +1,45 @@ dataset: blimp subset: principle_A_case_1 templates: - 037c9434-fc17-4122-97f8-0cbae4474e3b: !Template + 3eb3811b-66a1-4628-93fa-a9a73da66986: !Template + answer_choices: Yes ||| No + id: 3eb3811b-66a1-4628-93fa-a9a73da66986 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes + reference: '' + 44b2b73d-381e-43e4-9f73-dcf282979f5f: !Template answer_choices: A ||| B - id: 037c9434-fc17-4122-97f8-0cbae4474e3b + id: 44b2b73d-381e-43e4-9f73-dcf282979f5f jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -38,32 +53,31 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: better_A_B reference: '' - 053b6d4b-7920-4d03-aec0-caf6af2ae144: !Template - answer_choices: 1 ||| 2 - id: 053b6d4b-7920-4d03-aec0-caf6af2ae144 + 6bbdba99-6b3d-4cfa-a38d-274e44c0c67a: !Template + answer_choices: A ||| B + id: 6bbdba99-6b3d-4cfa-a38d-274e44c0c67a jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -77,42 +91,26 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 - reference: '' - 4860ae64-04b7-4e87-be92-1d8646dc0944: !Template - answer_choices: Yes ||| No - id: 4860ae64-04b7-4e87-be92-1d8646dc0944 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes + name: better_B_A reference: '' - 571c2f09-4dcd-496c-ae40-aaf9528124dd: !Template - answer_choices: 1 ||| 2 - id: 571c2f09-4dcd-496c-ae40-aaf9528124dd + 86a16480-2aea-4850-b44e-47d19f9b311d: !Template + answer_choices: A ||| B + id: 86a16480-2aea-4850-b44e-47d19f9b311d jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -132,31 +130,32 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: grammatical_which_one_A_B reference: '' - 601034e7-057a-44f6-91dc-c7fc1f13a1f9: !Template - answer_choices: A ||| B - id: 601034e7-057a-44f6-91dc-c7fc1f13a1f9 + a95609b9-6873-451a-90c1-7cfb2cfc8f2c: !Template + answer_choices: 1 ||| 2 + id: a95609b9-6873-451a-90c1-7cfb2cfc8f2c jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -170,11 +169,27 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: grammatical_which_one_1_2 reference: '' - 81a47751-1343-4e10-99c0-ad60337d0520: !Template + b690d38b-d4d2-46c6-98d8-030f7c36e500: !Template answer_choices: Yes ||| No - id: 81a47751-1343-4e10-99c0-ad60337d0520 + id: b690d38b-d4d2-46c6-98d8-030f7c36e500 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + bbdf5636-0581-48d2-8929-354f4dc35776: !Template + answer_choices: Yes ||| No + id: bbdf5636-0581-48d2-8929-354f4dc35776 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -188,24 +203,24 @@ templates: original_task: false name: single_sentence_bad_no_yes reference: '' - 9c059288-83e0-4cc1-b360-8dda049fbc0e: !Template - answer_choices: A ||| B - id: 9c059288-83e0-4cc1-b360-8dda049fbc0e + f2238a93-85a7-4ad7-b959-d9600b403a9b: !Template + answer_choices: 1 ||| 2 + id: f2238a93-85a7-4ad7-b959-d9600b403a9b jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -225,47 +240,32 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B - reference: '' - b6ddaee0-a4dd-4329-8182-720025ccef2d: !Template - answer_choices: Yes ||| No - id: b6ddaee0-a4dd-4329-8182-720025ccef2d - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no + name: grammatical_between_1_2 reference: '' - e04accc3-6f39-4446-861c-b7d2e87b0bfc: !Template + f30b527e-273e-46ff-9604-ff050fa978fb: !Template answer_choices: A ||| B - id: e04accc3-6f39-4446-861c-b7d2e87b0bfc + id: f30b527e-273e-46ff-9604-ff050fa978fb jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -279,11 +279,11 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: grammatical_between_A_B reference: '' - eb8a8935-0a2a-499d-a030-c1add8f0e5f9: !Template + fb1dd699-4d2f-462b-9365-2e0dc323cdad: !Template answer_choices: Yes ||| No - id: eb8a8935-0a2a-499d-a030-c1add8f0e5f9 + id: fb1dd699-4d2f-462b-9365-2e0dc323cdad jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" diff --git a/promptsource/templates/blimp/principle_A_case_2/templates.yaml b/promptsource/templates/blimp/principle_A_case_2/templates.yaml index 8d0d64f19..e1770bbc2 100644 --- a/promptsource/templates/blimp/principle_A_case_2/templates.yaml +++ b/promptsource/templates/blimp/principle_A_case_2/templates.yaml @@ -1,30 +1,29 @@ dataset: blimp subset: principle_A_case_2 templates: - 0355f72c-184e-4d7c-97b3-643fac97d800: !Template + 07a2c2fe-d131-4e00-b06e-c9f944ebc667: !Template answer_choices: A ||| B - id: 0355f72c-184e-4d7c-97b3-643fac97d800 + id: 07a2c2fe-d131-4e00-b06e-c9f944ebc667 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -38,32 +37,31 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: better_A_B reference: '' - 1b8c4d7c-e7ae-4053-9dd1-eafc41b509c0: !Template - answer_choices: 1 ||| 2 - id: 1b8c4d7c-e7ae-4053-9dd1-eafc41b509c0 + 1be73e9c-bd8e-46e6-b0ab-a3950aebafe6: !Template + answer_choices: A ||| B + id: 1be73e9c-bd8e-46e6-b0ab-a3950aebafe6 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -77,47 +75,32 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 - reference: '' - 3009bf8b-81ac-4cd2-b421-4ad701735004: !Template - answer_choices: Yes ||| No - id: 3009bf8b-81ac-4cd2-b421-4ad701735004 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no + name: better_B_A reference: '' - 38abf4cd-fb02-40a8-8c86-8b3f0d5aa007: !Template + 42f048db-e7cb-4984-9301-d703de4668ee: !Template answer_choices: A ||| B - id: 38abf4cd-fb02-40a8-8c86-8b3f0d5aa007 + id: 42f048db-e7cb-4984-9301-d703de4668ee jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -131,26 +114,42 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: grammatical_which_one_A_B reference: '' - 58866b5c-0625-487f-b859-0b968c29f745: !Template - answer_choices: A ||| B - id: 58866b5c-0625-487f-b859-0b968c29f745 + 4b17e2fe-920a-49c3-8ce6-417c8d3de8f4: !Template + answer_choices: Yes ||| No + id: 4b17e2fe-920a-49c3-8ce6-417c8d3de8f4 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes + reference: '' + 7cf6d5b8-f88f-4dee-80d0-4b839cbe4ff7: !Template + answer_choices: 1 ||| 2 + id: 7cf6d5b8-f88f-4dee-80d0-4b839cbe4ff7 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -170,11 +169,11 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: grammatical_between_1_2 reference: '' - 6d0da650-5594-4f61-9e61-112205cdad64: !Template + 8d2a603b-c0ab-455d-a8fa-f36f452a1d7b: !Template answer_choices: Yes ||| No - id: 6d0da650-5594-4f61-9e61-112205cdad64 + id: 8d2a603b-c0ab-455d-a8fa-f36f452a1d7b jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -188,9 +187,9 @@ templates: original_task: false name: single_sentence_good_yes_no reference: '' - a5e376d0-4d84-4763-8a22-13995232fb4d: !Template + ab66f18b-bcc9-4256-9a4e-dd71b7b5193f: !Template answer_choices: Yes ||| No - id: a5e376d0-4d84-4763-8a22-13995232fb4d + id: ab66f18b-bcc9-4256-9a4e-dd71b7b5193f jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -204,9 +203,9 @@ templates: original_task: false name: single_sentence_good_no_yes reference: '' - b149dd7a-1f6f-43db-beb0-0d926149576b: !Template + b13277a5-1402-41d5-af0e-fb9918efd902: !Template answer_choices: 1 ||| 2 - id: b149dd7a-1f6f-43db-beb0-0d926149576b + id: b13277a5-1402-41d5-af0e-fb9918efd902 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer 1 or 2. @@ -243,29 +242,46 @@ templates: original_task: true name: grammatical_which_one_1_2 reference: '' - bf408753-f905-40f8-9c58-9601f98ce7d7: !Template + c1d7dd5b-a830-4373-b9a9-70359f89ec40: !Template + answer_choices: Yes ||| No + id: c1d7dd5b-a830-4373-b9a9-70359f89ec40 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + eefd5af0-3c31-434d-8fa7-063247b47d04: !Template answer_choices: A ||| B - id: bf408753-f905-40f8-9c58-9601f98ce7d7 + id: eefd5af0-3c31-434d-8fa7-063247b47d04 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -279,21 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A - reference: '' - c52c20cc-369d-4d77-ad55-605f1d046255: !Template - answer_choices: Yes ||| No - id: c52c20cc-369d-4d77-ad55-605f1d046255 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes + name: grammatical_between_A_B reference: '' diff --git a/promptsource/templates/blimp/principle_A_domain_1/templates.yaml b/promptsource/templates/blimp/principle_A_domain_1/templates.yaml index 588865d41..ed9298fa3 100644 --- a/promptsource/templates/blimp/principle_A_domain_1/templates.yaml +++ b/promptsource/templates/blimp/principle_A_domain_1/templates.yaml @@ -1,12 +1,12 @@ dataset: blimp subset: principle_A_domain_1 templates: - 196cf1b1-28b6-4f4d-9e7c-51c7e2bbbd88: !Template + 3b145aa4-2436-487b-a7dc-8b5897b6fb41: !Template answer_choices: A ||| B - id: 196cf1b1-28b6-4f4d-9e7c-51c7e2bbbd88 + id: 3b145aa4-2436-487b-a7dc-8b5897b6fb41 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -38,30 +38,46 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: grammatical_between_A_B + reference: '' + 3f29d99e-8bef-477e-935e-faf920c15efe: !Template + answer_choices: Yes ||| No + id: 3f29d99e-8bef-477e-935e-faf920c15efe + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes reference: '' - 1c73db3f-859a-4a0a-b636-7ef0693ccb7e: !Template + 4998cbe8-a5b7-4055-acc0-12cb44a85cc3: !Template answer_choices: A ||| B - id: 1c73db3f-859a-4a0a-b636-7ef0693ccb7e + id: 4998cbe8-a5b7-4055-acc0-12cb44a85cc3 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + Sentence A: {{ sentence_good }} Sentence B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} Sentence B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order + Which one is a better sentence of English, B or A? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -76,11 +92,43 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: better_B_A reference: '' - 1d2a8265-0ea0-449d-bd83-b0f93650eafb: !Template + 4b2276f4-a44a-42af-871d-d4e4956cf8c2: !Template answer_choices: Yes ||| No - id: 1d2a8265-0ea0-449d-bd83-b0f93650eafb + id: 4b2276f4-a44a-42af-871d-d4e4956cf8c2 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 84ad15ba-205b-451c-95a6-2adde1c712b1: !Template + answer_choices: Yes ||| No + id: 84ad15ba-205b-451c-95a6-2adde1c712b1 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 9ff72b55-5376-4f53-84dc-ea2054350a8c: !Template + answer_choices: Yes ||| No + id: 9ff72b55-5376-4f53-84dc-ea2054350a8c jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -94,24 +142,24 @@ templates: original_task: false name: single_sentence_bad_no_yes reference: '' - 49bb9a16-e2dc-4ba2-81a3-3c47b5525f4d: !Template - answer_choices: 1 ||| 2 - id: 49bb9a16-e2dc-4ba2-81a3-3c47b5525f4d + ca28bbd1-7e74-474f-a905-6232fa7ba6fa: !Template + answer_choices: A ||| B + id: ca28bbd1-7e74-474f-a905-6232fa7ba6fa jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -131,32 +179,31 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: grammatical_which_one_A_B reference: '' - 5f6c71f1-f5c3-45bc-bc8d-79e79c85a346: !Template - answer_choices: 1 ||| 2 - id: 5f6c71f1-f5c3-45bc-bc8d-79e79c85a346 + d65a7429-4400-4758-9dcb-02340e62f7e5: !Template + answer_choices: A ||| B + id: d65a7429-4400-4758-9dcb-02340e62f7e5 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -170,31 +217,32 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: better_A_B reference: '' - 67e21963-a1e8-4e0c-b9f4-1a98e610c85f: !Template - answer_choices: A ||| B - id: 67e21963-a1e8-4e0c-b9f4-1a98e610c85f + e3f52395-7418-4d01-845d-d651b87e9a96: !Template + answer_choices: 1 ||| 2 + id: e3f52395-7418-4d01-845d-d651b87e9a96 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -208,74 +256,26 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A - reference: '' - 6bf676f9-4dfe-478d-b041-e31370e68576: !Template - answer_choices: Yes ||| No - id: 6bf676f9-4dfe-478d-b041-e31370e68576 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes - reference: '' - 6c8e382e-6614-4d86-b042-764fd98c60b5: !Template - answer_choices: Yes ||| No - id: 6c8e382e-6614-4d86-b042-764fd98c60b5 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no - reference: '' - af9fd8e7-9bdb-403b-8a8c-3015d644e410: !Template - answer_choices: Yes ||| No - id: af9fd8e7-9bdb-403b-8a8c-3015d644e410 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no + name: grammatical_between_1_2 reference: '' - bd9fc57d-129f-4ad3-a266-adbb6558201d: !Template - answer_choices: A ||| B - id: bd9fc57d-129f-4ad3-a266-adbb6558201d + eee44046-fd19-4476-a6b8-d1f66537f282: !Template + answer_choices: 1 ||| 2 + id: eee44046-fd19-4476-a6b8-d1f66537f282 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -295,5 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: grammatical_which_one_1_2 reference: '' diff --git a/promptsource/templates/blimp/principle_A_domain_2/templates.yaml b/promptsource/templates/blimp/principle_A_domain_2/templates.yaml index 080be694d..e6611417c 100644 --- a/promptsource/templates/blimp/principle_A_domain_2/templates.yaml +++ b/promptsource/templates/blimp/principle_A_domain_2/templates.yaml @@ -1,9 +1,9 @@ dataset: blimp subset: principle_A_domain_2 templates: - 1bc7e35a-0ef1-49df-b0e0-21a60b7620ef: !Template + 0d84877a-7c75-4802-9295-4474c6aa0748: !Template answer_choices: A ||| B - id: 1bc7e35a-0ef1-49df-b0e0-21a60b7620ef + id: 0d84877a-7c75-4802-9295-4474c6aa0748 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer A or B. @@ -40,24 +40,56 @@ templates: original_task: true name: grammatical_which_one_A_B reference: '' - 34a9cc3d-6c52-4c4e-96f2-033ddb285652: !Template - answer_choices: A ||| B - id: 34a9cc3d-6c52-4c4e-96f2-033ddb285652 + 439f4802-1aec-4191-813f-6477f2dda250: !Template + answer_choices: Yes ||| No + id: 439f4802-1aec-4191-813f-6477f2dda250 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes + reference: '' + 4dc27764-1806-440e-8fe0-6dbf45ba10e0: !Template + answer_choices: Yes ||| No + id: 4dc27764-1806-440e-8fe0-6dbf45ba10e0 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 6b97560e-878b-41ae-a94d-a70eb631ffe7: !Template + answer_choices: 1 ||| 2 + id: 6b97560e-878b-41ae-a94d-a70eb631ffe7 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -77,46 +109,30 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B - reference: '' - 43259012-399d-4838-96bd-d3e9824baf29: !Template - answer_choices: Yes ||| No - id: 43259012-399d-4838-96bd-d3e9824baf29 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes + name: grammatical_which_one_1_2 reference: '' - 46bbba41-aa08-447e-901b-f1de07ec3fff: !Template + 78a064b5-081c-4383-a0ed-4620e9f586c8: !Template answer_choices: A ||| B - id: 46bbba41-aa08-447e-901b-f1de07ec3fff + id: 78a064b5-081c-4383-a0ed-4620e9f586c8 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + Sentence A: {{ sentence_good }} Sentence B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} Sentence B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order + Which one is a better sentence of English, A or B? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -131,47 +147,32 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A - reference: '' - 719373b4-beba-4c4d-ad14-8d77c1e2b9e8: !Template - answer_choices: Yes ||| No - id: 719373b4-beba-4c4d-ad14-8d77c1e2b9e8 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no + name: better_A_B reference: '' - 75c53e28-0b7c-40cf-bac6-0b06c3a881bd: !Template - answer_choices: A ||| B - id: 75c53e28-0b7c-40cf-bac6-0b06c3a881bd + 9b8aa47f-d848-4ac3-b7eb-d0c24def596a: !Template + answer_choices: 1 ||| 2 + id: 9b8aa47f-d848-4ac3-b7eb-d0c24def596a jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -185,32 +186,31 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: grammatical_between_1_2 reference: '' - 89c6f7da-cf9e-4deb-b11a-67943c938d78: !Template - answer_choices: 1 ||| 2 - id: 89c6f7da-cf9e-4deb-b11a-67943c938d78 + 9c63204c-ea2d-45a6-a1c8-fb0ed98f97eb: !Template + answer_choices: A ||| B + id: 9c63204c-ea2d-45a6-a1c8-fb0ed98f97eb jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -224,11 +224,27 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: better_B_A reference: '' - a2ff9305-d52e-4506-a2fc-4708e8d66621: !Template + a8e59cea-16ba-406e-a634-5d73f688c99e: !Template answer_choices: Yes ||| No - id: a2ff9305-d52e-4506-a2fc-4708e8d66621 + id: a8e59cea-16ba-406e-a634-5d73f688c99e + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + b8f06ed1-ee37-42f3-a150-01f6bb13ea9e: !Template + answer_choices: Yes ||| No + id: b8f06ed1-ee37-42f3-a150-01f6bb13ea9e jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -242,24 +258,24 @@ templates: original_task: false name: single_sentence_bad_no_yes reference: '' - e66c21ed-fd59-45ee-8515-a8e729a3f352: !Template - answer_choices: 1 ||| 2 - id: e66c21ed-fd59-45ee-8515-a8e729a3f352 + c09fb1af-64d5-4070-954c-0b80ea07943e: !Template + answer_choices: A ||| B + id: c09fb1af-64d5-4070-954c-0b80ea07943e jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -279,21 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 - reference: '' - ec388750-a9cc-49a2-8cc1-1dd619ef8b01: !Template - answer_choices: Yes ||| No - id: ec388750-a9cc-49a2-8cc1-1dd619ef8b01 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no + name: grammatical_between_A_B reference: '' diff --git a/promptsource/templates/blimp/principle_A_domain_3/templates.yaml b/promptsource/templates/blimp/principle_A_domain_3/templates.yaml index 2c8cb11ee..0a48a8159 100644 --- a/promptsource/templates/blimp/principle_A_domain_3/templates.yaml +++ b/promptsource/templates/blimp/principle_A_domain_3/templates.yaml @@ -1,9 +1,9 @@ dataset: blimp subset: principle_A_domain_3 templates: - 08f945b2-0396-4b0e-a7e7-32678c2f4580: !Template + 07eccea8-220d-4c16-8e6e-68238346309b: !Template answer_choices: 1 ||| 2 - id: 08f945b2-0396-4b0e-a7e7-32678c2f4580 + id: 07eccea8-220d-4c16-8e6e-68238346309b jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -40,64 +40,9 @@ templates: original_task: true name: grammatical_between_1_2 reference: '' - 0a205494-c642-4158-af18-9e9693a1bfdc: !Template + 630d4dc2-1bfe-428d-af74-2a0d084627b2: !Template answer_choices: Yes ||| No - id: 0a205494-c642-4158-af18-9e9693a1bfdc - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes - reference: '' - 3b34c4b3-c1e2-4300-bb4f-ecdb4827aff9: !Template - answer_choices: A ||| B - id: 3b34c4b3-c1e2-4300-bb4f-ecdb4827aff9 - jinja: '{% set shuffled_order = [0, 1] | random %} - - Between the two sentences, which one is grammatical? Please answer A or B. - - {% if shuffled_order == 0 %} - - A: {{ sentence_good }} - - B: {{ sentence_bad }} - - {% else %} - - A: {{ sentence_bad }} - - B: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} - - {% endif %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: grammatical_between_A_B - reference: '' - 3b54e84f-ee8f-4331-b127-6db5144f9b5a: !Template - answer_choices: Yes ||| No - id: 3b54e84f-ee8f-4331-b127-6db5144f9b5a + id: 630d4dc2-1bfe-428d-af74-2a0d084627b2 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -111,45 +56,46 @@ templates: original_task: false name: single_sentence_good_no_yes reference: '' - 5802d66e-5745-4ab7-99e9-20a55bac70d4: !Template + 6427fbe6-0837-49ed-88e0-95bb0f100bb3: !Template answer_choices: Yes ||| No - id: 5802d66e-5745-4ab7-99e9-20a55bac70d4 + id: 6427fbe6-0837-49ed-88e0-95bb0f100bb3 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_good_yes_no + name: single_sentence_bad_yes_no reference: '' - 741102c7-568c-4cf4-b4d4-83689c1af124: !Template + 68ed5a47-46e3-451c-8df1-1c8dd7bce47f: !Template answer_choices: A ||| B - id: 741102c7-568c-4cf4-b4d4-83689c1af124 + id: 68ed5a47-46e3-451c-8df1-1c8dd7bce47f jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -163,11 +109,11 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: grammatical_which_one_A_B reference: '' - 99aebe4a-8bfd-4c28-bc6b-fdb1ab2a41d6: !Template + 7c3be9e9-98ee-4e72-a714-083ced37debf: !Template answer_choices: 1 ||| 2 - id: 99aebe4a-8bfd-4c28-bc6b-fdb1ab2a41d6 + id: 7c3be9e9-98ee-4e72-a714-083ced37debf jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer 1 or 2. @@ -204,12 +150,28 @@ templates: original_task: true name: grammatical_which_one_1_2 reference: '' - b296b798-0d53-46e9-80f1-c6c66860b38b: !Template + 8ea6b960-cfcd-4acd-af69-03b9a1a47252: !Template + answer_choices: Yes ||| No + id: 8ea6b960-cfcd-4acd-af69-03b9a1a47252 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes + reference: '' + 950e78ff-023c-401e-9866-ddacc2d10820: !Template answer_choices: A ||| B - id: b296b798-0d53-46e9-80f1-c6c66860b38b + id: 950e78ff-023c-401e-9866-ddacc2d10820 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -241,40 +203,40 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: grammatical_between_A_B reference: '' - c906716d-27af-4ee8-b0cc-e7137f5ee43b: !Template + bd438d35-ae7e-4fca-978a-02d990d3bbf8: !Template answer_choices: Yes ||| No - id: c906716d-27af-4ee8-b0cc-e7137f5ee43b + id: bd438d35-ae7e-4fca-978a-02d990d3bbf8 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_bad_yes_no + name: single_sentence_good_yes_no reference: '' - dba98473-151b-42f1-853f-db5eb8057936: !Template + c4a7f3eb-c4f5-4dac-bc0c-d6dda5da77ef: !Template answer_choices: A ||| B - id: dba98473-151b-42f1-853f-db5eb8057936 + id: c4a7f3eb-c4f5-4dac-bc0c-d6dda5da77ef jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + Sentence A: {{ sentence_good }} Sentence B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} Sentence B: {{ sentence_good }} @@ -297,3 +259,41 @@ templates: original_task: true name: better_B_A reference: '' + c5f8ee18-8137-4af8-b086-6330963739f9: !Template + answer_choices: A ||| B + id: c5f8ee18-8137-4af8-b086-6330963739f9 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: better_A_B + reference: '' diff --git a/promptsource/templates/blimp/principle_A_reconstruction/templates.yaml b/promptsource/templates/blimp/principle_A_reconstruction/templates.yaml index 60afe26c0..17583ced8 100644 --- a/promptsource/templates/blimp/principle_A_reconstruction/templates.yaml +++ b/promptsource/templates/blimp/principle_A_reconstruction/templates.yaml @@ -1,9 +1,9 @@ dataset: blimp subset: principle_A_reconstruction templates: - 3d9f42c6-bd69-4cb8-9bb3-44d2cc95c6cf: !Template + 255e6923-6c36-4bbb-86a6-3e9aaa1f048a: !Template answer_choices: Yes ||| No - id: 3d9f42c6-bd69-4cb8-9bb3-44d2cc95c6cf + id: 255e6923-6c36-4bbb-86a6-3e9aaa1f048a jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -17,13 +17,13 @@ templates: original_task: false name: single_sentence_bad_yes_no reference: '' - 76b92be2-35c7-4b4b-85a1-3b71f3f9a0aa: !Template + 28235ddb-4ecb-41d4-b571-179ef4be2934: !Template answer_choices: Yes ||| No - id: 76b92be2-35c7-4b4b-85a1-3b71f3f9a0aa + id: 28235ddb-4ecb-41d4-b571-179ef4be2934 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -31,32 +31,31 @@ templates: metrics: - Accuracy original_task: false - name: single_sentence_good_yes_no + name: single_sentence_good_no_yes reference: '' - 92401ec1-563a-4b64-9dc7-a96529aafc9b: !Template + 299bf6ee-117a-4e38-91b2-6c3fe9043f85: !Template answer_choices: A ||| B - id: 92401ec1-563a-4b64-9dc7-a96529aafc9b + id: 299bf6ee-117a-4e38-91b2-6c3fe9043f85 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -70,26 +69,42 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: better_B_A reference: '' - 9e15325b-a26b-4fbc-b92b-4ca4ec48d283: !Template - answer_choices: 1 ||| 2 - id: 9e15325b-a26b-4fbc-b92b-4ca4ec48d283 + 3c781311-5236-4805-b6f1-90bb44da84cd: !Template + answer_choices: Yes ||| No + id: 3c781311-5236-4805-b6f1-90bb44da84cd + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes + reference: '' + 3d6382bf-026b-4448-8c14-ee7296731e48: !Template + answer_choices: A ||| B + id: 3d6382bf-026b-4448-8c14-ee7296731e48 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -109,42 +124,26 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 - reference: '' - b3f82dfc-421d-470d-ba43-c21ebbfbed3c: !Template - answer_choices: Yes ||| No - id: b3f82dfc-421d-470d-ba43-c21ebbfbed3c - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes + name: grammatical_between_A_B reference: '' - c0392d6b-4b76-439b-8683-d5a50ba97b9e: !Template - answer_choices: A ||| B - id: c0392d6b-4b76-439b-8683-d5a50ba97b9e + 3e90b2cb-0b87-446a-9ff7-e6f0378101c8: !Template + answer_choices: 1 ||| 2 + id: 3e90b2cb-0b87-446a-9ff7-e6f0378101c8 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -164,31 +163,32 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: grammatical_which_one_1_2 reference: '' - c64d706d-53d8-4d9e-b468-7f80f8cc787b: !Template + 902acbd1-3493-4ea3-8cef-fa8942784645: !Template answer_choices: A ||| B - id: c64d706d-53d8-4d9e-b468-7f80f8cc787b + id: 902acbd1-3493-4ea3-8cef-fa8942784645 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -202,24 +202,24 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: grammatical_which_one_A_B reference: '' - e1eb7e07-3be2-4c15-9e0d-b1f39dd40eb2: !Template + bd2bca31-aadd-4544-b8e8-7519cb1438f5: !Template answer_choices: A ||| B - id: e1eb7e07-3be2-4c15-9e0d-b1f39dd40eb2 + id: bd2bca31-aadd-4544-b8e8-7519cb1438f5 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + Sentence A: {{ sentence_good }} Sentence B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} Sentence B: {{ sentence_good }} @@ -242,12 +242,28 @@ templates: original_task: true name: better_A_B reference: '' - e27d8725-2082-4b0a-a759-44dd4921396b: !Template + bf05357a-eac9-46b1-8063-614519e87174: !Template + answer_choices: Yes ||| No + id: bf05357a-eac9-46b1-8063-614519e87174 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + d4734cfb-f96d-418e-9def-e0a2beccdb33: !Template answer_choices: 1 ||| 2 - id: e27d8725-2082-4b0a-a759-44dd4921396b + id: d4734cfb-f96d-418e-9def-e0a2beccdb33 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -279,21 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 - reference: '' - e8ceafa4-e73d-476b-bbfc-abf93d867e23: !Template - answer_choices: Yes ||| No - id: e8ceafa4-e73d-476b-bbfc-abf93d867e23 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes + name: grammatical_between_1_2 reference: '' diff --git a/promptsource/templates/blimp/regular_plural_subject_verb_agreement_1/templates.yaml b/promptsource/templates/blimp/regular_plural_subject_verb_agreement_1/templates.yaml index aafdb829b..23e6a168e 100644 --- a/promptsource/templates/blimp/regular_plural_subject_verb_agreement_1/templates.yaml +++ b/promptsource/templates/blimp/regular_plural_subject_verb_agreement_1/templates.yaml @@ -1,29 +1,62 @@ dataset: blimp subset: regular_plural_subject_verb_agreement_1 templates: - 022a7339-fa4f-4177-8f64-d8b232d4f8f2: !Template - answer_choices: A ||| B - id: 022a7339-fa4f-4177-8f64-d8b232d4f8f2 + 03829374-4f10-4144-91cb-16eb4a00495d: !Template + answer_choices: Yes ||| No + id: 03829374-4f10-4144-91cb-16eb4a00495d + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 1eae7b0b-42e9-4fea-9270-e35cb3280f23: !Template + answer_choices: Yes ||| No + id: 1eae7b0b-42e9-4fea-9270-e35cb3280f23 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 26fa366b-d7d5-42bb-b5f2-7c58135286d3: !Template + answer_choices: 1 ||| 2 + id: 26fa366b-d7d5-42bb-b5f2-7c58135286d3 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -37,32 +70,31 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: grammatical_which_one_1_2 reference: '' - 248db1ea-7f34-4b5e-98a1-990fa3f2602e: !Template + 3a5e64f9-3ae3-4104-a0a4-0f92a6a9a5c9: !Template answer_choices: A ||| B - id: 248db1ea-7f34-4b5e-98a1-990fa3f2602e + id: 3a5e64f9-3ae3-4104-a0a4-0f92a6a9a5c9 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -76,11 +108,11 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: better_B_A reference: '' - 3d7a95a5-cb60-460b-b1de-bee81fcb600e: !Template + 5c7da74e-fb56-4846-950d-a0a934751260: !Template answer_choices: 1 ||| 2 - id: 3d7a95a5-cb60-460b-b1de-bee81fcb600e + id: 5c7da74e-fb56-4846-950d-a0a934751260 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -117,28 +149,12 @@ templates: original_task: true name: grammatical_between_1_2 reference: '' - 47884791-2028-460c-8c7a-71c05a606cf3: !Template - answer_choices: Yes ||| No - id: 47884791-2028-460c-8c7a-71c05a606cf3 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes - reference: '' - 6fc714a5-6a0c-4938-af80-297c63864f8d: !Template + 60c569bb-0814-4ece-9c68-78982b58c812: !Template answer_choices: A ||| B - id: 6fc714a5-6a0c-4938-af80-297c63864f8d + id: 60c569bb-0814-4ece-9c68-78982b58c812 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -170,74 +186,26 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B - reference: '' - 7b27d408-68b0-4440-aa36-a6ad32ddec8c: !Template - answer_choices: Yes ||| No - id: 7b27d408-68b0-4440-aa36-a6ad32ddec8c - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no - reference: '' - 8a7b76ea-5b2e-4a5d-930c-8498372cca63: !Template - answer_choices: Yes ||| No - id: 8a7b76ea-5b2e-4a5d-930c-8498372cca63 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no - reference: '' - 8bcadc10-d2eb-4258-829a-90203e050487: !Template - answer_choices: Yes ||| No - id: 8bcadc10-d2eb-4258-829a-90203e050487 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes + name: grammatical_which_one_A_B reference: '' - ac11f4e8-8b7e-4f4e-99f9-2e0242484ce2: !Template - answer_choices: 1 ||| 2 - id: ac11f4e8-8b7e-4f4e-99f9-2e0242484ce2 + a95dfe01-aa62-4bbc-a133-5c629d0a8f13: !Template + answer_choices: A ||| B + id: a95dfe01-aa62-4bbc-a133-5c629d0a8f13 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -257,30 +225,30 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: grammatical_between_A_B reference: '' - b0ff3c16-d083-48bc-8074-bc9ab25caa9a: !Template + b2642c64-3fdf-4cd9-a36a-add92f0f0b25: !Template answer_choices: A ||| B - id: b0ff3c16-d083-48bc-8074-bc9ab25caa9a + id: b2642c64-3fdf-4cd9-a36a-add92f0f0b25 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + Sentence A: {{ sentence_good }} Sentence B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} Sentence B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order + Which one is a better sentence of English, A or B? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -295,5 +263,37 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: better_A_B + reference: '' + e1c58f46-2f51-4e1c-b4ca-99d8b1df59ec: !Template + answer_choices: Yes ||| No + id: e1c58f46-2f51-4e1c-b4ca-99d8b1df59ec + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes + reference: '' + e48d87f9-47f2-44ab-9c5f-4a1c05e48d1e: !Template + answer_choices: Yes ||| No + id: e48d87f9-47f2-44ab-9c5f-4a1c05e48d1e + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes reference: '' diff --git a/promptsource/templates/blimp/regular_plural_subject_verb_agreement_2/templates.yaml b/promptsource/templates/blimp/regular_plural_subject_verb_agreement_2/templates.yaml index 7999a0684..a28497131 100644 --- a/promptsource/templates/blimp/regular_plural_subject_verb_agreement_2/templates.yaml +++ b/promptsource/templates/blimp/regular_plural_subject_verb_agreement_2/templates.yaml @@ -1,9 +1,9 @@ dataset: blimp subset: regular_plural_subject_verb_agreement_2 templates: - 039cee11-ac01-4aba-8667-a001c4a66428: !Template + 03ee8117-9f09-409c-89cb-0d5901a6fa42: !Template answer_choices: 1 ||| 2 - id: 039cee11-ac01-4aba-8667-a001c4a66428 + id: 03ee8117-9f09-409c-89cb-0d5901a6fa42 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer 1 or 2. @@ -40,22 +40,22 @@ templates: original_task: true name: grammatical_which_one_1_2 reference: '' - 5c94d850-6a7a-4ed4-bc48-3a432b21f194: !Template + 09ddee17-3c41-4343-85d5-78f3766deb52: !Template answer_choices: A ||| B - id: 5c94d850-6a7a-4ed4-bc48-3a432b21f194 + id: 09ddee17-3c41-4343-85d5-78f3766deb52 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + Sentence A: {{ sentence_good }} Sentence B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} Sentence B: {{ sentence_good }} @@ -78,25 +78,25 @@ templates: original_task: true name: better_A_B reference: '' - 5cde73e5-5556-49e4-aed7-b0e80e38f92d: !Template + 3807b338-f2b9-437f-8d96-c3b045cad368: !Template answer_choices: Yes ||| No - id: 5cde73e5-5556-49e4-aed7-b0e80e38f92d + id: 3807b338-f2b9-437f-8d96-c3b045cad368 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_good_no_yes + name: single_sentence_bad_yes_no reference: '' - 6586899d-4afd-425f-b4a0-eb7f1e7b4d33: !Template + 5474d660-bfd3-40d0-ab81-8ba59b0d9763: !Template answer_choices: A ||| B - id: 6586899d-4afd-425f-b4a0-eb7f1e7b4d33 + id: 5474d660-bfd3-40d0-ab81-8ba59b0d9763 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer A or B. @@ -133,61 +133,22 @@ templates: original_task: true name: grammatical_between_A_B reference: '' - bc0fd9b9-e5b3-4e66-b991-d9be54fda16b: !Template - answer_choices: 1 ||| 2 - id: bc0fd9b9-e5b3-4e66-b991-d9be54fda16b - jinja: '{% set shuffled_order = [0, 1] | random %} - - Between the two sentences, which one is grammatical? Please answer 1 or 2. - - {% if shuffled_order == 0 %} - - 1: {{ sentence_good }} - - 2: {{ sentence_bad }} - - {% else %} - - 1: {{ sentence_bad }} - - 2: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} - - {% endif %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: grammatical_between_1_2 - reference: '' - dac79382-484c-4581-9075-e5691fae76b7: !Template + 87c5d723-7a5f-4764-8a79-099dea11505f: !Template answer_choices: A ||| B - id: dac79382-484c-4581-9075-e5691fae76b7 + id: 87c5d723-7a5f-4764-8a79-099dea11505f jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + Sentence A: {{ sentence_good }} Sentence B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} Sentence B: {{ sentence_good }} @@ -210,9 +171,9 @@ templates: original_task: true name: better_B_A reference: '' - dfe5bb12-4134-4615-9547-2ee980c784f0: !Template + a17707c6-a358-4e15-89b0-4451c6a7e800: !Template answer_choices: Yes ||| No - id: dfe5bb12-4134-4615-9547-2ee980c784f0 + id: a17707c6-a358-4e15-89b0-4451c6a7e800 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -226,41 +187,48 @@ templates: original_task: false name: single_sentence_bad_no_yes reference: '' - e0f5744e-c31a-4ae1-aa6c-a92b82fc3a1e: !Template - answer_choices: Yes ||| No - id: e0f5744e-c31a-4ae1-aa6c-a92b82fc3a1e - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + b71b2409-7828-4888-8577-87f09a67b414: !Template + answer_choices: 1 ||| 2 + id: b71b2409-7828-4888-8577-87f09a67b414 + jinja: '{% set shuffled_order = [0, 1] | random %} - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no - reference: '' - ea7aa88c-8ad2-4cc0-8af3-ef647f7bd173: !Template - answer_choices: Yes ||| No - id: ea7aa88c-8ad2-4cc0-8af3-ef647f7bd173 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + Between the two sentences, which one is grammatical? Please answer 1 or 2. - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: false - name: single_sentence_bad_yes_no + original_task: true + name: grammatical_between_1_2 reference: '' - f44231dc-4eee-4e9d-945c-70225b6e0c43: !Template + cc6fb3df-3728-4ba7-bd87-a4f92c22aebf: !Template answer_choices: A ||| B - id: f44231dc-4eee-4e9d-945c-70225b6e0c43 + id: cc6fb3df-3728-4ba7-bd87-a4f92c22aebf jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer A or B. @@ -297,3 +265,35 @@ templates: original_task: true name: grammatical_which_one_A_B reference: '' + e629bf12-9c79-4d68-9e26-517c471bd68b: !Template + answer_choices: Yes ||| No + id: e629bf12-9c79-4d68-9e26-517c471bd68b + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + ee810cef-0734-4df6-bb54-9ea41683b943: !Template + answer_choices: Yes ||| No + id: ee810cef-0734-4df6-bb54-9ea41683b943 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes + reference: '' diff --git a/promptsource/templates/blimp/sentential_negation_npi_licensor_present/templates.yaml b/promptsource/templates/blimp/sentential_negation_npi_licensor_present/templates.yaml index d9450a20f..6874ae16d 100644 --- a/promptsource/templates/blimp/sentential_negation_npi_licensor_present/templates.yaml +++ b/promptsource/templates/blimp/sentential_negation_npi_licensor_present/templates.yaml @@ -1,12 +1,28 @@ dataset: blimp subset: sentential_negation_npi_licensor_present templates: - 20f8866d-81c6-4eb0-b9ae-0d0028c78e77: !Template + 15e6a065-a3fb-4a23-a5c7-682853907265: !Template + answer_choices: Yes ||| No + id: 15e6a065-a3fb-4a23-a5c7-682853907265 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 1ef568c1-a567-4c97-a284-6478c447a4f9: !Template answer_choices: 1 ||| 2 - id: 20f8866d-81c6-4eb0-b9ae-0d0028c78e77 + id: 1ef568c1-a567-4c97-a284-6478c447a4f9 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -38,47 +54,32 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 - reference: '' - 49cd246c-4a62-431c-b4a6-efbd75a05284: !Template - answer_choices: Yes ||| No - id: 49cd246c-4a62-431c-b4a6-efbd75a05284 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes + name: grammatical_between_1_2 reference: '' - 4d2b0ac9-8c1c-4577-b5b4-018ca2e4cf7b: !Template - answer_choices: A ||| B - id: 4d2b0ac9-8c1c-4577-b5b4-018ca2e4cf7b + 21ac6dfd-1678-4b3e-a017-4274c37a1e99: !Template + answer_choices: 1 ||| 2 + id: 21ac6dfd-1678-4b3e-a017-4274c37a1e99 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -92,32 +93,31 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: grammatical_which_one_1_2 reference: '' - 519cf39e-5aee-489a-90ee-bdbcb55d0e71: !Template + 44f93830-67ca-4c38-8ec4-9c1d05762e52: !Template answer_choices: A ||| B - id: 519cf39e-5aee-489a-90ee-bdbcb55d0e71 + id: 44f93830-67ca-4c38-8ec4-9c1d05762e52 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -131,26 +131,26 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: better_B_A reference: '' - 5a1dfa62-42cb-4be9-a0ae-a29ba2242514: !Template - answer_choices: 1 ||| 2 - id: 5a1dfa62-42cb-4be9-a0ae-a29ba2242514 + 759dd383-5fa6-4966-ad16-6037ed96cdb6: !Template + answer_choices: A ||| B + id: 759dd383-5fa6-4966-ad16-6037ed96cdb6 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -170,47 +170,32 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 - reference: '' - 5f904965-a73a-4128-8749-32e36fd8ec1c: !Template - answer_choices: Yes ||| No - id: 5f904965-a73a-4128-8749-32e36fd8ec1c - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no + name: grammatical_which_one_A_B reference: '' - 634e3e9a-d706-4555-891f-38389954cec8: !Template + b1cea338-a852-4e95-b12b-6e34c4102afa: !Template answer_choices: A ||| B - id: 634e3e9a-d706-4555-891f-38389954cec8 + id: b1cea338-a852-4e95-b12b-6e34c4102afa jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -224,64 +209,63 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: grammatical_between_A_B reference: '' - dbd72c8a-e8c0-4804-a1dc-9d29aa54c325: !Template + ca401a85-9a0c-4152-b2a7-7a46687acf75: !Template answer_choices: Yes ||| No - id: dbd72c8a-e8c0-4804-a1dc-9d29aa54c325 + id: ca401a85-9a0c-4152-b2a7-7a46687acf75 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_bad_yes_no + name: single_sentence_good_no_yes reference: '' - f67c53ae-ef19-4ccf-b608-68dd85fc279c: !Template + de077405-43ec-4b5c-b1e4-8ef8a9b7726e: !Template answer_choices: Yes ||| No - id: f67c53ae-ef19-4ccf-b608-68dd85fc279c + id: de077405-43ec-4b5c-b1e4-8ef8a9b7726e jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_good_no_yes + name: single_sentence_bad_no_yes reference: '' - fa8140a0-a58c-48f1-883a-ab755fa17b37: !Template + e9632649-6761-4648-a785-989545485a48: !Template answer_choices: A ||| B - id: fa8140a0-a58c-48f1-883a-ab755fa17b37 + id: e9632649-6761-4648-a785-989545485a48 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -295,5 +279,21 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: better_A_B + reference: '' + ea8d173a-11a8-4755-b9a4-1425c00af1d5: !Template + answer_choices: Yes ||| No + id: ea8d173a-11a8-4755-b9a4-1425c00af1d5 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no reference: '' diff --git a/promptsource/templates/blimp/sentential_negation_npi_scope/templates.yaml b/promptsource/templates/blimp/sentential_negation_npi_scope/templates.yaml index 147f0ddcb..f2833ac3c 100644 --- a/promptsource/templates/blimp/sentential_negation_npi_scope/templates.yaml +++ b/promptsource/templates/blimp/sentential_negation_npi_scope/templates.yaml @@ -1,9 +1,9 @@ dataset: blimp subset: sentential_negation_npi_scope templates: - 0be0526d-3819-49e6-8411-ab62400de1e5: !Template + 63e93f22-a757-4861-8581-5d22d4fedf10: !Template answer_choices: Yes ||| No - id: 0be0526d-3819-49e6-8411-ab62400de1e5 + id: 63e93f22-a757-4861-8581-5d22d4fedf10 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -17,29 +17,46 @@ templates: original_task: false name: single_sentence_good_no_yes reference: '' - 0c9cdd4a-8ee4-4d49-bae6-e878cc85b1fb: !Template - answer_choices: A ||| B - id: 0c9cdd4a-8ee4-4d49-bae6-e878cc85b1fb + 6a6def12-011c-47eb-a54b-0ebc1ecdd062: !Template + answer_choices: Yes ||| No + id: 6a6def12-011c-47eb-a54b-0ebc1ecdd062 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 6d87f4c2-8e7e-4653-a9fd-e5b2edd0d0da: !Template + answer_choices: 1 ||| 2 + id: 6d87f4c2-8e7e-4653-a9fd-e5b2edd0d0da jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -53,11 +70,11 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: grammatical_which_one_1_2 reference: '' - 127e191c-83da-43a1-b5b4-0a2ba26e8808: !Template + 7691fb0c-048a-49ce-815f-910736e57d2f: !Template answer_choices: 1 ||| 2 - id: 127e191c-83da-43a1-b5b4-0a2ba26e8808 + id: 7691fb0c-048a-49ce-815f-910736e57d2f jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -94,13 +111,13 @@ templates: original_task: true name: grammatical_between_1_2 reference: '' - 232a94b1-2f09-40c2-a89a-a3d497153252: !Template + 9810324c-5934-43eb-a6af-00bda7decb2a: !Template answer_choices: Yes ||| No - id: 232a94b1-2f09-40c2-a89a-a3d497153252 + id: 9810324c-5934-43eb-a6af-00bda7decb2a jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata @@ -108,32 +125,31 @@ templates: metrics: - Accuracy original_task: false - name: single_sentence_bad_no_yes + name: single_sentence_bad_yes_no reference: '' - 2b237dd0-d5f7-4781-9ff4-49330b863849: !Template + a27d0b6d-0f6b-4d3f-aa90-4a76eb9d1c93: !Template answer_choices: A ||| B - id: 2b237dd0-d5f7-4781-9ff4-49330b863849 + id: a27d0b6d-0f6b-4d3f-aa90-4a76eb9d1c93 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -147,32 +163,31 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: better_B_A reference: '' - 2b6a009d-9e14-42bc-98ce-825218c7e89c: !Template + ae8dd951-4b8d-4122-b62f-2f541fe0aa63: !Template answer_choices: A ||| B - id: 2b6a009d-9e14-42bc-98ce-825218c7e89c + id: ae8dd951-4b8d-4122-b62f-2f541fe0aa63 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -186,31 +201,32 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: better_A_B reference: '' - 5fb90498-24a5-45d3-a65b-11e45b51f69c: !Template + d4e8dbdc-91f7-479a-b66c-0a709aa08d7c: !Template answer_choices: A ||| B - id: 5fb90498-24a5-45d3-a65b-11e45b51f69c + id: d4e8dbdc-91f7-479a-b66c-0a709aa08d7c jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -224,15 +240,15 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: grammatical_which_one_A_B reference: '' - 76134d63-66d5-4c72-8e0f-6fbed4c6e363: !Template + efb2fec6-0f4c-454c-88ea-d54326c27d1a: !Template answer_choices: Yes ||| No - id: 76134d63-66d5-4c72-8e0f-6fbed4c6e363 + id: efb2fec6-0f4c-454c-88ea-d54326c27d1a jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata @@ -240,26 +256,26 @@ templates: metrics: - Accuracy original_task: false - name: single_sentence_bad_yes_no + name: single_sentence_bad_no_yes reference: '' - abaf5b42-4106-4327-9431-a3000475c589: !Template - answer_choices: 1 ||| 2 - id: abaf5b42-4106-4327-9431-a3000475c589 + fef4e9a5-b9aa-4cd7-bfaa-9d9bbc4956a1: !Template + answer_choices: A ||| B + id: fef4e9a5-b9aa-4cd7-bfaa-9d9bbc4956a1 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -279,21 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 - reference: '' - bfe1e980-cb40-4cac-97f9-a7acc923c8ea: !Template - answer_choices: Yes ||| No - id: bfe1e980-cb40-4cac-97f9-a7acc923c8ea - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no + name: grammatical_between_A_B reference: '' diff --git a/promptsource/templates/blimp/sentential_subject_island/templates.yaml b/promptsource/templates/blimp/sentential_subject_island/templates.yaml index 32fbda974..1671b143f 100644 --- a/promptsource/templates/blimp/sentential_subject_island/templates.yaml +++ b/promptsource/templates/blimp/sentential_subject_island/templates.yaml @@ -1,40 +1,24 @@ dataset: blimp subset: sentential_subject_island templates: - 057724f1-d841-4441-a7e0-4147de41894a: !Template - answer_choices: Yes ||| No - id: 057724f1-d841-4441-a7e0-4147de41894a - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no - reference: '' - 37f96a2a-25ec-4a71-b435-510d112c60a2: !Template - answer_choices: 1 ||| 2 - id: 37f96a2a-25ec-4a71-b435-510d112c60a2 + 01c97538-2dd1-4950-b762-e980947844b5: !Template + answer_choices: A ||| B + id: 01c97538-2dd1-4950-b762-e980947844b5 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -54,26 +38,26 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: grammatical_between_A_B reference: '' - 3ef04d7b-83b1-45f6-ba19-5c992e177b11: !Template - answer_choices: A ||| B - id: 3ef04d7b-83b1-45f6-ba19-5c992e177b11 + 2f70d0e5-6ecb-4dcb-afa4-7dd32665eeaa: !Template + answer_choices: 1 ||| 2 + id: 2f70d0e5-6ecb-4dcb-afa4-7dd32665eeaa jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -93,48 +77,31 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B - reference: '' - 45674b88-247c-4d4f-89c2-34ae33616ad0: !Template - answer_choices: Yes ||| No - id: 45674b88-247c-4d4f-89c2-34ae33616ad0 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes + name: grammatical_which_one_1_2 reference: '' - 73864b31-6e30-48b8-91a8-7a4fead2cebf: !Template - answer_choices: 1 ||| 2 - id: 73864b31-6e30-48b8-91a8-7a4fead2cebf + 43d6b55a-70f3-4336-8c0f-915b44c86a01: !Template + answer_choices: A ||| B + id: 43d6b55a-70f3-4336-8c0f-915b44c86a01 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -148,24 +115,24 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: better_B_A reference: '' - 785928f5-c9bb-449c-be3a-e2c7ddd00f3f: !Template + 715ea764-29c6-4bf5-941e-b7189a6f5fce: !Template answer_choices: A ||| B - id: 785928f5-c9bb-449c-be3a-e2c7ddd00f3f + id: 715ea764-29c6-4bf5-941e-b7189a6f5fce jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + Sentence A: {{ sentence_good }} Sentence B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} Sentence B: {{ sentence_good }} @@ -188,40 +155,24 @@ templates: original_task: true name: better_A_B reference: '' - 81cf5f0f-fbe8-4ede-98ec-d2b8de738d20: !Template - answer_choices: Yes ||| No - id: 81cf5f0f-fbe8-4ede-98ec-d2b8de738d20 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes - reference: '' - 9bed3ec0-4e1b-4f7a-a222-655af91c2cf2: !Template - answer_choices: A ||| B - id: 9bed3ec0-4e1b-4f7a-a222-655af91c2cf2 + a54d6794-30cf-43b6-930f-e521581243fc: !Template + answer_choices: 1 ||| 2 + id: a54d6794-30cf-43b6-930f-e521581243fc jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -241,47 +192,48 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: grammatical_between_1_2 reference: '' - dc7f52e0-3880-430f-b3bc-fb85b9688de4: !Template + ac00abca-5ad0-4498-b2c7-7915205e5207: !Template answer_choices: Yes ||| No - id: dc7f52e0-3880-430f-b3bc-fb85b9688de4 + id: ac00abca-5ad0-4498-b2c7-7915205e5207 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_bad_yes_no + name: single_sentence_good_no_yes reference: '' - fecd0f72-b5ce-424b-a155-389da926817a: !Template + bdbe0233-1eaf-47ed-af52-718c16a8a966: !Template answer_choices: A ||| B - id: fecd0f72-b5ce-424b-a155-389da926817a + id: bdbe0233-1eaf-47ed-af52-718c16a8a966 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -295,5 +247,53 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: grammatical_which_one_A_B + reference: '' + bdc0016f-41c3-45ff-b109-fe514ff65a19: !Template + answer_choices: Yes ||| No + id: bdc0016f-41c3-45ff-b109-fe514ff65a19 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + ebbc3bb5-02cc-4d3a-8a24-44d1faf7cdf7: !Template + answer_choices: Yes ||| No + id: ebbc3bb5-02cc-4d3a-8a24-44d1faf7cdf7 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes + reference: '' + f666c04d-183c-42dc-85f8-992dab34ec3f: !Template + answer_choices: Yes ||| No + id: f666c04d-183c-42dc-85f8-992dab34ec3f + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no reference: '' diff --git a/promptsource/templates/blimp/superlative_quantifiers_1/templates.yaml b/promptsource/templates/blimp/superlative_quantifiers_1/templates.yaml index c72c7130b..424c2d954 100644 --- a/promptsource/templates/blimp/superlative_quantifiers_1/templates.yaml +++ b/promptsource/templates/blimp/superlative_quantifiers_1/templates.yaml @@ -1,29 +1,30 @@ dataset: blimp subset: superlative_quantifiers_1 templates: - 2a069f15-0c13-431e-8447-ecc471255aa8: !Template - answer_choices: A ||| B - id: 2a069f15-0c13-431e-8447-ecc471255aa8 + 16215d7d-0159-4f11-bd52-e44d6ec7627a: !Template + answer_choices: 1 ||| 2 + id: 16215d7d-0159-4f11-bd52-e44d6ec7627a jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -37,32 +38,31 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: grammatical_between_1_2 reference: '' - 2a3bb462-021d-46b0-94ee-a382d54ea3bc: !Template - answer_choices: 1 ||| 2 - id: 2a3bb462-021d-46b0-94ee-a382d54ea3bc + 3b474c05-3dd0-4052-a311-c7b93bd00953: !Template + answer_choices: A ||| B + id: 3b474c05-3dd0-4052-a311-c7b93bd00953 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -76,27 +76,27 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: better_B_A reference: '' - 2c870ca2-d51d-45c9-acee-7d92deefd54c: !Template + 42f68a66-02b9-4010-b3da-756cf61553c8: !Template answer_choices: Yes ||| No - id: 2c870ca2-d51d-45c9-acee-7d92deefd54c + id: 42f68a66-02b9-4010-b3da-756cf61553c8 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_bad_no_yes + name: single_sentence_good_yes_no reference: '' - 35f1d980-1a3d-4407-a731-d7ec801f594e: !Template + 48fd0552-beca-4ad7-812b-324960109f33: !Template answer_choices: A ||| B - id: 35f1d980-1a3d-4407-a731-d7ec801f594e + id: 48fd0552-beca-4ad7-812b-324960109f33 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer A or B. @@ -133,46 +133,45 @@ templates: original_task: true name: grammatical_which_one_A_B reference: '' - 3eeabe94-4b00-465a-810a-e05e63a62a84: !Template + 5985d362-e21b-4066-8edb-db530a01d25d: !Template answer_choices: Yes ||| No - id: 3eeabe94-4b00-465a-810a-e05e63a62a84 + id: 5985d362-e21b-4066-8edb-db530a01d25d jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_bad_yes_no + name: single_sentence_good_no_yes reference: '' - 4e5615fa-52dc-49fe-a0e2-56236b4e733d: !Template - answer_choices: 1 ||| 2 - id: 4e5615fa-52dc-49fe-a0e2-56236b4e733d + abe584d4-28dd-4797-ae93-09aa5c6508d8: !Template + answer_choices: A ||| B + id: abe584d4-28dd-4797-ae93-09aa5c6508d8 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -186,47 +185,32 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 - reference: '' - 84abb131-62a8-4f62-b76e-b4d53807fa75: !Template - answer_choices: Yes ||| No - id: 84abb131-62a8-4f62-b76e-b4d53807fa75 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes + name: better_A_B reference: '' - 917ef9e0-02ab-40c4-a126-5177a02523ea: !Template - answer_choices: A ||| B - id: 917ef9e0-02ab-40c4-a126-5177a02523ea + b096d7eb-ac86-4773-bb28-11c18402a04b: !Template + answer_choices: 1 ||| 2 + id: b096d7eb-ac86-4773-bb28-11c18402a04b jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -240,11 +224,43 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: grammatical_which_one_1_2 + reference: '' + b8aa07e6-d21b-41a7-9443-7b4cace14b58: !Template + answer_choices: Yes ||| No + id: b8aa07e6-d21b-41a7-9443-7b4cace14b58 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + bc904949-593e-41a1-9b6b-f034ddc14cee: !Template + answer_choices: Yes ||| No + id: bc904949-593e-41a1-9b6b-f034ddc14cee + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes reference: '' - 9cc03745-3cdf-4e84-bf41-47dd4d114ee0: !Template + e5d7b9ab-5949-40ab-a1b2-438f708e9206: !Template answer_choices: A ||| B - id: 9cc03745-3cdf-4e84-bf41-47dd4d114ee0 + id: e5d7b9ab-5949-40ab-a1b2-438f708e9206 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer A or B. @@ -281,19 +297,3 @@ templates: original_task: true name: grammatical_between_A_B reference: '' - 9ce8c6ac-061c-47ce-aff3-43ba68d41081: !Template - answer_choices: Yes ||| No - id: 9ce8c6ac-061c-47ce-aff3-43ba68d41081 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no - reference: '' diff --git a/promptsource/templates/blimp/superlative_quantifiers_2/templates.yaml b/promptsource/templates/blimp/superlative_quantifiers_2/templates.yaml index c708f7074..ca3b8056e 100644 --- a/promptsource/templates/blimp/superlative_quantifiers_2/templates.yaml +++ b/promptsource/templates/blimp/superlative_quantifiers_2/templates.yaml @@ -1,9 +1,25 @@ dataset: blimp subset: superlative_quantifiers_2 templates: - 2aae800b-a6cd-46fc-a518-fbc5ef126337: !Template + 12d1a32b-a052-4afd-9ffa-4d035926bd28: !Template answer_choices: Yes ||| No - id: 2aae800b-a6cd-46fc-a518-fbc5ef126337 + id: 12d1a32b-a052-4afd-9ffa-4d035926bd28 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes + reference: '' + 18bf1579-5a8d-429b-8111-6fd2485c4a8a: !Template + answer_choices: Yes ||| No + id: 18bf1579-5a8d-429b-8111-6fd2485c4a8a jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -17,9 +33,9 @@ templates: original_task: false name: single_sentence_good_yes_no reference: '' - 2f4e0d4c-0a1e-4a70-aadb-a8793def2ed6: !Template + 281f71ef-6c15-4a2b-955b-0d29d27e209d: !Template answer_choices: Yes ||| No - id: 2f4e0d4c-0a1e-4a70-aadb-a8793def2ed6 + id: 281f71ef-6c15-4a2b-955b-0d29d27e209d jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -33,12 +49,12 @@ templates: original_task: false name: single_sentence_bad_no_yes reference: '' - 342f2622-44f7-4df2-900c-1bff71b37dad: !Template + 3b6d68ca-ec93-4c48-be59-c70d76395226: !Template answer_choices: A ||| B - id: 342f2622-44f7-4df2-900c-1bff71b37dad + id: 3b6d68ca-ec93-4c48-be59-c70d76395226 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -70,24 +86,24 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: grammatical_which_one_A_B reference: '' - 3ab2784f-f409-4041-9472-c454bc26b0cb: !Template + 62142866-a88e-409c-a00c-5430bc02a17e: !Template answer_choices: A ||| B - id: 3ab2784f-f409-4041-9472-c454bc26b0cb + id: 62142866-a88e-409c-a00c-5430bc02a17e jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + Sentence A: {{ sentence_good }} Sentence B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} Sentence B: {{ sentence_good }} @@ -110,46 +126,29 @@ templates: original_task: true name: better_B_A reference: '' - 7a7242c8-f0aa-4a44-8b0e-9a0c098e3504: !Template - answer_choices: Yes ||| No - id: 7a7242c8-f0aa-4a44-8b0e-9a0c098e3504 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes - reference: '' - 7c7ceb51-7cbf-48d2-9fe7-0e5f56d2767b: !Template + 6606bf9f-0678-4feb-b3b5-74496ad49387: !Template answer_choices: A ||| B - id: 7c7ceb51-7cbf-48d2-9fe7-0e5f56d2767b + id: 6606bf9f-0678-4feb-b3b5-74496ad49387 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -163,11 +162,27 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: better_A_B + reference: '' + 68c9bcd3-a7b3-43cb-af83-4356b029cd12: !Template + answer_choices: Yes ||| No + id: 68c9bcd3-a7b3-43cb-af83-4356b029cd12 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no reference: '' - 7e4ec882-cd79-478d-86f5-68ca8f726343: !Template + 775f4a9c-5955-46a7-90b6-9690745b0aa5: !Template answer_choices: 1 ||| 2 - id: 7e4ec882-cd79-478d-86f5-68ca8f726343 + id: 775f4a9c-5955-46a7-90b6-9690745b0aa5 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer 1 or 2. @@ -204,24 +219,24 @@ templates: original_task: true name: grammatical_which_one_1_2 reference: '' - a03a7186-d11b-4a9a-8a6f-edb965803b0e: !Template - answer_choices: 1 ||| 2 - id: a03a7186-d11b-4a9a-8a6f-edb965803b0e + a6f91c04-05ef-48e7-abbc-4d3aae328e7b: !Template + answer_choices: A ||| B + id: a6f91c04-05ef-48e7-abbc-4d3aae328e7b jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -241,47 +256,32 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 - reference: '' - b13b6751-8ef0-427a-8a85-67302fdcaaac: !Template - answer_choices: Yes ||| No - id: b13b6751-8ef0-427a-8a85-67302fdcaaac - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no + name: grammatical_between_A_B reference: '' - e358818b-d4e2-4b13-b471-1b3efa8db9e7: !Template - answer_choices: A ||| B - id: e358818b-d4e2-4b13-b471-1b3efa8db9e7 + e68f5f96-b39a-4153-a9cb-7f08f5ccf235: !Template + answer_choices: 1 ||| 2 + id: e68f5f96-b39a-4153-a9cb-7f08f5ccf235 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -295,5 +295,5 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: grammatical_between_1_2 reference: '' diff --git a/promptsource/templates/blimp/tough_vs_raising_1/templates.yaml b/promptsource/templates/blimp/tough_vs_raising_1/templates.yaml index 1487c95c4..73f2e8b0f 100644 --- a/promptsource/templates/blimp/tough_vs_raising_1/templates.yaml +++ b/promptsource/templates/blimp/tough_vs_raising_1/templates.yaml @@ -1,50 +1,28 @@ dataset: blimp subset: tough_vs_raising_1 templates: - 11f1d6e4-fecb-43e6-82e1-d1bcb9d42b60: !Template - answer_choices: A ||| B - id: 11f1d6e4-fecb-43e6-82e1-d1bcb9d42b60 - jinja: '{% set shuffled_order = [0, 1] | random %} - - Read the following two sentences. - - {% if shuffled_order == 0 %} - - Sentence A: {{ sentence_good }} - - Sentence B: {{ sentence_bad }} - - {% else %} - - Sentence A: {{ sentence_bad }} - - Sentence B: {{ sentence_good }} - - {% endif %} - - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} + 14d15e3b-d9c2-4470-a886-b125ca6625d7: !Template + answer_choices: Yes ||| No + id: 14d15e3b-d9c2-4470-a886-b125ca6625d7 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - {% endif %}' + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: better_B_A + original_task: false + name: single_sentence_good_no_yes reference: '' - 4f4787f9-a285-4438-b7ad-a83aabfde1d3: !Template + 471a0e2a-dcc5-46c3-ba4c-0fdb16801dbf: !Template answer_choices: A ||| B - id: 4f4787f9-a285-4438-b7ad-a83aabfde1d3 + id: 471a0e2a-dcc5-46c3-ba4c-0fdb16801dbf jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -76,32 +54,47 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: grammatical_which_one_A_B reference: '' - 6f925c37-203b-431c-86e5-f7e96ffaaae6: !Template - answer_choices: 1 ||| 2 - id: 6f925c37-203b-431c-86e5-f7e96ffaaae6 + 7c683c93-8b88-47f8-ab71-5cc6757cb336: !Template + answer_choices: Yes ||| No + id: 7c683c93-8b88-47f8-ab71-5cc6757cb336 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 920f31d5-b4b9-4d0f-a010-1025b8f960c0: !Template + answer_choices: A ||| B + id: 920f31d5-b4b9-4d0f-a010-1025b8f960c0 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -115,30 +108,30 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: better_B_A reference: '' - 71893fff-522d-4cb3-8fef-6d34f6fd969b: !Template + ae9fb107-6313-419c-a478-ab002e17450b: !Template answer_choices: Yes ||| No - id: 71893fff-522d-4cb3-8fef-6d34f6fd969b + id: ae9fb107-6313-419c-a478-ab002e17450b jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_bad_yes_no + name: single_sentence_good_yes_no reference: '' - 7debf4a9-23ca-45fa-a67d-2103ae2336a9: !Template + d3552793-a5a3-4c64-9630-9e167fcd58f6: !Template answer_choices: 1 ||| 2 - id: 7debf4a9-23ca-45fa-a67d-2103ae2336a9 + id: d3552793-a5a3-4c64-9630-9e167fcd58f6 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -170,26 +163,26 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: grammatical_which_one_1_2 reference: '' - 7e869d1a-29e8-43cf-b4e9-1e128d5acd95: !Template - answer_choices: A ||| B - id: 7e869d1a-29e8-43cf-b4e9-1e128d5acd95 + d918a306-025a-4b85-a3f5-ba8777488120: !Template + answer_choices: 1 ||| 2 + id: d918a306-025a-4b85-a3f5-ba8777488120 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -209,40 +202,24 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B - reference: '' - b5c49924-ca7f-43b2-86b1-588a5d6915e2: !Template - answer_choices: Yes ||| No - id: b5c49924-ca7f-43b2-86b1-588a5d6915e2 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes + name: grammatical_between_1_2 reference: '' - cb656ca1-9b29-4598-9231-7ba2c793565b: !Template + de63fc9f-d970-4435-aca7-e4351920bf17: !Template answer_choices: A ||| B - id: cb656ca1-9b29-4598-9231-7ba2c793565b + id: de63fc9f-d970-4435-aca7-e4351920bf17 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + Sentence A: {{ sentence_good }} Sentence B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} Sentence B: {{ sentence_good }} @@ -265,9 +242,9 @@ templates: original_task: true name: better_A_B reference: '' - cfd001eb-0f98-4f91-afac-6151e6ea3294: !Template + e0ec6447-53e2-48cf-a9de-7b1e539a55f0: !Template answer_choices: Yes ||| No - id: cfd001eb-0f98-4f91-afac-6151e6ea3294 + id: e0ec6447-53e2-48cf-a9de-7b1e539a55f0 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -281,19 +258,42 @@ templates: original_task: false name: single_sentence_bad_no_yes reference: '' - e1b6ebf4-b1eb-4a96-a644-a5c918cf85f8: !Template - answer_choices: Yes ||| No - id: e1b6ebf4-b1eb-4a96-a644-a5c918cf85f8 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + f4074fdb-d1c5-457b-8a53-f6140338db36: !Template + answer_choices: A ||| B + id: f4074fdb-d1c5-457b-8a53-f6140338db36 + jinja: '{% set shuffled_order = [0, 1] | random %} - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: false - name: single_sentence_good_yes_no + original_task: true + name: grammatical_between_A_B reference: '' diff --git a/promptsource/templates/blimp/tough_vs_raising_2/templates.yaml b/promptsource/templates/blimp/tough_vs_raising_2/templates.yaml index 0ea200f30..28c5aecaa 100644 --- a/promptsource/templates/blimp/tough_vs_raising_2/templates.yaml +++ b/promptsource/templates/blimp/tough_vs_raising_2/templates.yaml @@ -1,22 +1,22 @@ dataset: blimp subset: tough_vs_raising_2 templates: - 05b504b6-1c19-4083-ad26-cecc6afd1363: !Template + 001c0904-7795-4bcf-bcfe-d26b8b335e7f: !Template answer_choices: A ||| B - id: 05b504b6-1c19-4083-ad26-cecc6afd1363 + id: 001c0904-7795-4bcf-bcfe-d26b8b335e7f jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + Sentence A: {{ sentence_good }} Sentence B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} Sentence B: {{ sentence_good }} @@ -39,12 +39,28 @@ templates: original_task: true name: better_B_A reference: '' - 13dedd40-8c4a-45b7-949c-2acd071b1821: !Template + 022c1683-8607-4e99-9570-3a9524f30cf6: !Template + answer_choices: Yes ||| No + id: 022c1683-8607-4e99-9570-3a9524f30cf6 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 5cef6dec-0c90-4d69-bda4-f73a02536190: !Template answer_choices: 1 ||| 2 - id: 13dedd40-8c4a-45b7-949c-2acd071b1821 + id: 5cef6dec-0c90-4d69-bda4-f73a02536190 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -76,48 +92,31 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 - reference: '' - 1ca42122-f025-4a5d-8f58-d2c9771a55fa: !Template - answer_choices: Yes ||| No - id: 1ca42122-f025-4a5d-8f58-d2c9771a55fa - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes + name: grammatical_which_one_1_2 reference: '' - 273ca188-a5d2-4cb5-a9d2-6c5adc8ae087: !Template + 60617f99-d83c-474c-92ef-9c7f69b1b56b: !Template answer_choices: A ||| B - id: 273ca188-a5d2-4cb5-a9d2-6c5adc8ae087 + id: 60617f99-d83c-474c-92ef-9c7f69b1b56b jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -131,47 +130,48 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: better_A_B reference: '' - 357d62f1-fb86-48c3-a1b0-69c71b5e5049: !Template + 7d3f7996-76cf-4cb6-ac39-829e697eb9c2: !Template answer_choices: Yes ||| No - id: 357d62f1-fb86-48c3-a1b0-69c71b5e5049 + id: 7d3f7996-76cf-4cb6-ac39-829e697eb9c2 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_good_yes_no + name: single_sentence_bad_no_yes reference: '' - 97f22748-e724-4f87-a17f-2a7a6aae80d0: !Template + 7d92c27a-05aa-49a8-b37a-a4ecc8e0590a: !Template answer_choices: A ||| B - id: 97f22748-e724-4f87-a17f-2a7a6aae80d0 + id: 7d92c27a-05aa-49a8-b37a-a4ecc8e0590a jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -185,27 +185,27 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: grammatical_which_one_A_B reference: '' - ac5a8915-c5bb-449e-aa5f-4cb9a422876a: !Template + 90962099-c490-4a7c-9c9c-be0074b2ace5: !Template answer_choices: Yes ||| No - id: ac5a8915-c5bb-449e-aa5f-4cb9a422876a + id: 90962099-c490-4a7c-9c9c-be0074b2ace5 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_bad_yes_no + name: single_sentence_good_no_yes reference: '' - c83e59e0-b51a-4bcb-9bfb-baae15b0b9d2: !Template + 9909addf-0c61-439a-9886-ac5c9b5fccf2: !Template answer_choices: A ||| B - id: c83e59e0-b51a-4bcb-9bfb-baae15b0b9d2 + id: 9909addf-0c61-439a-9886-ac5c9b5fccf2 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer A or B. @@ -242,28 +242,12 @@ templates: original_task: true name: grammatical_between_A_B reference: '' - ede13f24-7d23-49b8-8d00-83cb4b2a99e2: !Template - answer_choices: Yes ||| No - id: ede13f24-7d23-49b8-8d00-83cb4b2a99e2 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes - reference: '' - f5bc4220-7cac-451a-9886-440911243a54: !Template + d24ae7a3-9e0e-4f4a-8359-5b28872effc1: !Template answer_choices: 1 ||| 2 - id: f5bc4220-7cac-451a-9886-440911243a54 + id: d24ae7a3-9e0e-4f4a-8359-5b28872effc1 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -295,5 +279,21 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: grammatical_between_1_2 + reference: '' + de5df7ee-9d96-41b6-ab45-c277be0847c9: !Template + answer_choices: Yes ||| No + id: de5df7ee-9d96-41b6-ab45-c277be0847c9 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no reference: '' From 12466453ca6e97ad9a7abc9212e4d4aaac5028cd Mon Sep 17 00:00:00 2001 From: najoungkim Date: Wed, 27 Apr 2022 16:42:35 -0400 Subject: [PATCH 14/16] Added null prompts (true null and single quotation mark versions) --- promptsource/templates.py | 1 + .../blimp/adjunct_island/templates.yaml | 22 ++ .../anaphor_gender_agreement/templates.yaml | 202 ++++++++------ .../anaphor_number_agreement/templates.yaml | 210 +++++++------- .../animate_subject_passive/templates.yaml | 166 ++++++----- .../animate_subject_trans/templates.yaml | 232 +++++++++------- .../templates/blimp/causative/templates.yaml | 216 ++++++++------- .../blimp/complex_NP_island/templates.yaml | 200 ++++++++------ .../templates.yaml | 246 +++++++++-------- .../templates.yaml | 214 ++++++++------- .../templates.yaml | 220 ++++++++------- .../templates.yaml | 206 +++++++------- .../templates.yaml | 238 ++++++++-------- .../templates.yaml | 212 +++++++------- .../templates.yaml | 182 ++++++------ .../templates.yaml | 198 ++++++++------ .../templates.yaml | 210 +++++++------- .../templates.yaml | 204 ++++++++------ .../templates.yaml | 234 +++++++++------- .../templates.yaml | 208 +++++++------- .../blimp/drop_argument/templates.yaml | 226 ++++++++------- .../blimp/ellipsis_n_bar_1/templates.yaml | 174 ++++++------ .../blimp/ellipsis_n_bar_2/templates.yaml | 208 +++++++------- .../templates.yaml | 160 ++++++----- .../templates.yaml | 156 ++++++----- .../templates.yaml | 214 ++++++++------- .../templates.yaml | 206 +++++++------- .../templates.yaml | 200 ++++++++------ .../templates/blimp/inchoative/templates.yaml | 196 +++++++------ .../blimp/intransitive/templates.yaml | 178 ++++++------ .../templates.yaml | 190 +++++++------ .../templates.yaml | 236 ++++++++-------- .../templates.yaml | 220 ++++++++------- .../templates.yaml | 210 +++++++------- .../templates.yaml | 256 +++++++++-------- .../templates.yaml | 218 ++++++++------- .../templates.yaml | 222 ++++++++------- .../blimp/npi_present_1/templates.yaml | 184 +++++++------ .../blimp/npi_present_2/templates.yaml | 202 ++++++++------ .../only_npi_licensor_present/templates.yaml | 224 ++++++++------- .../blimp/only_npi_scope/templates.yaml | 198 ++++++++------ .../templates/blimp/passive_1/templates.yaml | 232 +++++++++------- .../templates/blimp/passive_2/templates.yaml | 172 +++++++----- .../principle_A_c_command/templates.yaml | 218 ++++++++------- .../blimp/principle_A_case_1/templates.yaml | 212 +++++++------- .../blimp/principle_A_case_2/templates.yaml | 206 +++++++------- .../blimp/principle_A_domain_1/templates.yaml | 146 +++++----- .../blimp/principle_A_domain_2/templates.yaml | 240 ++++++++-------- .../blimp/principle_A_domain_3/templates.yaml | 210 +++++++------- .../principle_A_reconstruction/templates.yaml | 222 ++++++++------- .../templates.yaml | 228 +++++++++------- .../templates.yaml | 218 ++++++++------- .../templates.yaml | 168 +++++++----- .../templates.yaml | 226 ++++++++------- .../sentential_subject_island/templates.yaml | 222 ++++++++------- .../superlative_quantifiers_1/templates.yaml | 190 +++++++------ .../superlative_quantifiers_2/templates.yaml | 194 +++++++------ .../blimp/tough_vs_raising_1/templates.yaml | 192 +++++++------ .../blimp/tough_vs_raising_2/templates.yaml | 228 +++++++++------- .../templates/blimp/transitive/templates.yaml | 188 +++++++------ .../templates/blimp/wh_island/templates.yaml | 224 ++++++++------- .../wh_questions_object_gap/templates.yaml | 234 +++++++++------- .../wh_questions_subject_gap/templates.yaml | 232 +++++++++------- .../templates.yaml | 208 +++++++------- .../blimp/wh_vs_that_no_gap/templates.yaml | 210 +++++++------- .../templates.yaml | 258 ++++++++++-------- .../blimp/wh_vs_that_with_gap/templates.yaml | 194 +++++++------ .../templates.yaml | 224 ++++++++------- 68 files changed, 7635 insertions(+), 6160 deletions(-) diff --git a/promptsource/templates.py b/promptsource/templates.py index 3ece927fe..02353c203 100644 --- a/promptsource/templates.py +++ b/promptsource/templates.py @@ -380,6 +380,7 @@ def read_from_file(self) -> Dict: """ if not os.path.exists(self.yaml_path): + print('============yaml path============', self.yaml_path) dataset_name = f"{self.dataset_name} {self.subset_name}" if self.subset_name else self.dataset_name logging.warning( f"Tried instantiating `DatasetTemplates` for {dataset_name}, but no prompts found. " diff --git a/promptsource/templates/blimp/adjunct_island/templates.yaml b/promptsource/templates/blimp/adjunct_island/templates.yaml index 8bb424bc2..ff863c633 100644 --- a/promptsource/templates/blimp/adjunct_island/templates.yaml +++ b/promptsource/templates/blimp/adjunct_island/templates.yaml @@ -78,6 +78,17 @@ templates: original_task: true name: grammatical_between_1_2 reference: '' + 77a180c8-9529-4675-88ea-3b50cb9a093d: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 77a180c8-9529-4675-88ea-3b50cb9a093d + jinja: '" ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' 7ed733fd-524c-4f00-800f-77a7a862ab4a: !Template answer_choices: 1 ||| 2 id: 7ed733fd-524c-4f00-800f-77a7a862ab4a @@ -297,3 +308,14 @@ templates: original_task: false name: single_sentence_good_no_yes reference: '' + f6cfe60f-a0c5-4ab8-ae7d-1c08d3cdaa2b: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: f6cfe60f-a0c5-4ab8-ae7d-1c08d3cdaa2b + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: true_null_prompt + reference: '' diff --git a/promptsource/templates/blimp/anaphor_gender_agreement/templates.yaml b/promptsource/templates/blimp/anaphor_gender_agreement/templates.yaml index 0f904a6c3..b26c0b22a 100644 --- a/promptsource/templates/blimp/anaphor_gender_agreement/templates.yaml +++ b/promptsource/templates/blimp/anaphor_gender_agreement/templates.yaml @@ -1,24 +1,24 @@ dataset: blimp subset: anaphor_gender_agreement templates: - 14410b9a-0ae9-4e49-b2ec-cdce6a8fba37: !Template - answer_choices: A ||| B - id: 14410b9a-0ae9-4e49-b2ec-cdce6a8fba37 + 28c2b5aa-29ce-4d36-aa80-daf33f3f5c57: !Template + answer_choices: 1 ||| 2 + id: 28c2b5aa-29ce-4d36-aa80-daf33f3f5c57 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -38,64 +38,31 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B - reference: '' - 27066b83-d53e-4260-ac95-e4ba390c78c3: !Template - answer_choices: Yes ||| No - id: 27066b83-d53e-4260-ac95-e4ba390c78c3 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes - reference: '' - 289dda3f-3052-4851-bd8c-d53a71355a3c: !Template - answer_choices: Yes ||| No - id: 289dda3f-3052-4851-bd8c-d53a71355a3c - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes + name: grammatical_which_one_1_2 reference: '' - 302c2622-e23b-422b-8821-6bfa410e0038: !Template - answer_choices: 1 ||| 2 - id: 302c2622-e23b-422b-8821-6bfa410e0038 + 2bc4df5e-6432-494f-a88a-9a22e1c0ab6b: !Template + answer_choices: A ||| B + id: 2bc4df5e-6432-494f-a88a-9a22e1c0ab6b jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -109,11 +76,27 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: better_B_A + reference: '' + 3188a3e3-5cb9-4f03-9087-ef289c16ec68: !Template + answer_choices: Yes ||| No + id: 3188a3e3-5cb9-4f03-9087-ef289c16ec68 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes reference: '' - 38a12baf-18de-4fac-8689-3e2c5fcdbd74: !Template + 4b3a2ad2-494f-4a3f-b296-9fe185edc1d8: !Template answer_choices: A ||| B - id: 38a12baf-18de-4fac-8689-3e2c5fcdbd74 + id: 4b3a2ad2-494f-4a3f-b296-9fe185edc1d8 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -132,7 +115,7 @@ templates: {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order + Which one is a better sentence of English, A or B? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -147,11 +130,11 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: better_A_B reference: '' - 4abf6023-679a-4f9f-9a21-0ddb167b7bbd: !Template + 6c675909-285a-4ed8-89bb-da2e40b288ee: !Template answer_choices: A ||| B - id: 4abf6023-679a-4f9f-9a21-0ddb167b7bbd + id: 6c675909-285a-4ed8-89bb-da2e40b288ee jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer A or B. @@ -188,44 +171,39 @@ templates: original_task: true name: grammatical_which_one_A_B reference: '' - 5da96864-24ae-44a2-93b5-9e4f4734d5de: !Template - answer_choices: Yes ||| No - id: 5da96864-24ae-44a2-93b5-9e4f4734d5de - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + 849851e4-f1f9-4c01-b25d-abe588438454: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 849851e4-f1f9-4c01-b25d-abe588438454 + jinja: ' ||| {{ sentence_good }}' metadata: !TemplateMetadata - choices_in_prompt: true + choices_in_prompt: false metrics: - Accuracy - original_task: false - name: single_sentence_good_yes_no + original_task: true + name: true_null_prompt reference: '' - a3767823-ab46-4099-aad7-54a66b819210: !Template + 8fd53d64-b95b-45cb-87fc-c9e765bc8b70: !Template answer_choices: Yes ||| No - id: a3767823-ab46-4099-aad7-54a66b819210 + id: 8fd53d64-b95b-45cb-87fc-c9e765bc8b70 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_bad_yes_no + name: single_sentence_good_yes_no reference: '' - e111fc31-a73b-407d-997a-ba1c50cee471: !Template + b50e7d9c-e69b-4827-8333-96a7f5f76bd1: !Template answer_choices: 1 ||| 2 - id: e111fc31-a73b-407d-997a-ba1c50cee471 + id: b50e7d9c-e69b-4827-8333-96a7f5f76bd1 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -257,31 +235,48 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: grammatical_between_1_2 reference: '' - f209365f-af86-407a-bc9e-e609bcb8ae39: !Template + ca10bff1-9a6c-4281-90b2-23d65916afed: !Template + answer_choices: Yes ||| No + id: ca10bff1-9a6c-4281-90b2-23d65916afed + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes + reference: '' + d2eacc09-0036-4dda-adeb-905d891ad306: !Template answer_choices: A ||| B - id: f209365f-af86-407a-bc9e-e609bcb8ae39 + id: d2eacc09-0036-4dda-adeb-905d891ad306 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -295,5 +290,32 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: grammatical_between_A_B + reference: '' + e98d6ef5-acc7-4b5e-b064-a0834f9b084b: !Template + answer_choices: Yes ||| No + id: e98d6ef5-acc7-4b5e-b064-a0834f9b084b + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + eb865335-7712-4775-b3ea-76c61e7f15d5: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: eb865335-7712-4775-b3ea-76c61e7f15d5 + jinja: '" ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt reference: '' diff --git a/promptsource/templates/blimp/anaphor_number_agreement/templates.yaml b/promptsource/templates/blimp/anaphor_number_agreement/templates.yaml index 50e81f13a..a8e644105 100644 --- a/promptsource/templates/blimp/anaphor_number_agreement/templates.yaml +++ b/promptsource/templates/blimp/anaphor_number_agreement/templates.yaml @@ -1,12 +1,12 @@ dataset: blimp subset: anaphor_number_agreement templates: - 1678bdf4-83cf-4427-9a26-420a114697d5: !Template + 0847e8b0-ff2b-4a5f-bd0b-f071c7c3688b: !Template answer_choices: A ||| B - id: 1678bdf4-83cf-4427-9a26-420a114697d5 + id: 0847e8b0-ff2b-4a5f-bd0b-f071c7c3688b jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -38,27 +38,38 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: grammatical_between_A_B reference: '' - 2a55a588-c014-425d-8c84-f4fadcc158d6: !Template + 637b6e3b-d0fc-4090-8fb6-7ae52709bcdc: !Template answer_choices: Yes ||| No - id: 2a55a588-c014-425d-8c84-f4fadcc158d6 + id: 637b6e3b-d0fc-4090-8fb6-7ae52709bcdc jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_good_yes_no + name: single_sentence_bad_yes_no reference: '' - 2aa1369b-91a7-4c58-905b-f9ae34b8b9a1: !Template + 6af9b651-b6f4-4fa3-8c8e-2dc942f7d249: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 6af9b651-b6f4-4fa3-8c8e-2dc942f7d249 + jinja: '" ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + 6afa5333-ba67-4c5d-a67f-bd00c3cf8a52: !Template answer_choices: A ||| B - id: 2aa1369b-91a7-4c58-905b-f9ae34b8b9a1 + id: 6afa5333-ba67-4c5d-a67f-bd00c3cf8a52 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -77,7 +88,7 @@ templates: {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order + Which one is a better sentence of English, B or A? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -92,26 +103,69 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: better_B_A reference: '' - 2e05b4df-bf2f-4e00-ab53-7ab454b0b325: !Template - answer_choices: A ||| B - id: 2e05b4df-bf2f-4e00-ab53-7ab454b0b325 + 8e19ca7b-f2ca-4aa9-bb91-363ab48690c8: !Template + answer_choices: Yes ||| No + id: 8e19ca7b-f2ca-4aa9-bb91-363ab48690c8 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes + reference: '' + a654a3ba-a74e-4fd8-90ed-baba061f170b: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: a654a3ba-a74e-4fd8-90ed-baba061f170b + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: true_null_prompt + reference: '' + a6679425-6cd8-42e7-a7dd-1564a07b457c: !Template + answer_choices: Yes ||| No + id: a6679425-6cd8-42e7-a7dd-1564a07b457c + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes + reference: '' + ab0505c1-af35-44af-a874-7a579576109a: !Template + answer_choices: 1 ||| 2 + id: ab0505c1-af35-44af-a874-7a579576109a jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -131,27 +185,11 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B - reference: '' - b2735fbf-80ea-4f88-adb2-70cedec7779c: !Template - answer_choices: Yes ||| No - id: b2735fbf-80ea-4f88-adb2-70cedec7779c - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no + name: grammatical_which_one_1_2 reference: '' - b780c5a5-c0e6-49dd-bb18-7d04ea6a4ed2: !Template + b3efa878-3d7c-45bb-9a30-b6c5025a2945: !Template answer_choices: 1 ||| 2 - id: b780c5a5-c0e6-49dd-bb18-7d04ea6a4ed2 + id: b3efa878-3d7c-45bb-9a30-b6c5025a2945 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -188,61 +226,30 @@ templates: original_task: true name: grammatical_between_1_2 reference: '' - d58cd3c3-ec83-49df-8af5-ba2407dc9b98: !Template - answer_choices: Yes ||| No - id: d58cd3c3-ec83-49df-8af5-ba2407dc9b98 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes - reference: '' - e645a293-3597-49d0-ac78-b5f6a8a20f39: !Template - answer_choices: Yes ||| No - id: e645a293-3597-49d0-ac78-b5f6a8a20f39 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes - reference: '' - e7cd3e1a-a079-478d-82c8-6c04552e63cf: !Template + e39925fa-2aea-4ec9-81c5-b4a3137eef05: !Template answer_choices: A ||| B - id: e7cd3e1a-a079-478d-82c8-6c04552e63cf + id: e39925fa-2aea-4ec9-81c5-b4a3137eef05 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -256,32 +263,47 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: grammatical_which_one_A_B reference: '' - e9493f58-3944-4ed7-930b-75aeebe63d72: !Template - answer_choices: 1 ||| 2 - id: e9493f58-3944-4ed7-930b-75aeebe63d72 + e5f07808-fdd3-4bcd-a6ed-5c7af1d670de: !Template + answer_choices: Yes ||| No + id: e5f07808-fdd3-4bcd-a6ed-5c7af1d670de + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + f2edd012-b456-4fbb-9a61-035b16262057: !Template + answer_choices: A ||| B + id: f2edd012-b456-4fbb-9a61-035b16262057 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -295,5 +317,5 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: better_A_B reference: '' diff --git a/promptsource/templates/blimp/animate_subject_passive/templates.yaml b/promptsource/templates/blimp/animate_subject_passive/templates.yaml index fc8b37e87..4aea1fb90 100644 --- a/promptsource/templates/blimp/animate_subject_passive/templates.yaml +++ b/promptsource/templates/blimp/animate_subject_passive/templates.yaml @@ -1,9 +1,25 @@ dataset: blimp subset: animate_subject_passive templates: - 31f32f78-b308-45a6-bdfe-6855dcc6d8c6: !Template + 37db01ee-31a2-46d3-8f05-2b3d19a410d6: !Template answer_choices: Yes ||| No - id: 31f32f78-b308-45a6-bdfe-6855dcc6d8c6 + id: 37db01ee-31a2-46d3-8f05-2b3d19a410d6 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 3fbf0301-1963-4846-8515-8c8169eda731: !Template + answer_choices: Yes ||| No + id: 3fbf0301-1963-4846-8515-8c8169eda731 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -17,24 +33,24 @@ templates: original_task: false name: single_sentence_good_no_yes reference: '' - 5026b9ec-21d4-494d-80e1-2331080f44fd: !Template - answer_choices: A ||| B - id: 5026b9ec-21d4-494d-80e1-2331080f44fd + 40ccc24a-7616-45c6-b20a-f34a7a5e5952: !Template + answer_choices: 1 ||| 2 + id: 40ccc24a-7616-45c6-b20a-f34a7a5e5952 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -54,48 +70,58 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: grammatical_between_1_2 reference: '' - 6babaa62-0772-48db-80d4-c2cbc5671243: !Template + 5780ce16-31b2-4f46-8869-b30649be4a29: !Template answer_choices: Yes ||| No - id: 6babaa62-0772-48db-80d4-c2cbc5671243 + id: 5780ce16-31b2-4f46-8869-b30649be4a29 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_good_yes_no + name: single_sentence_bad_yes_no reference: '' - 88257b30-8c10-42f8-9df7-90a85431b575: !Template - answer_choices: 1 ||| 2 - id: 88257b30-8c10-42f8-9df7-90a85431b575 + 6721356a-6e94-43f1-8433-7629cc3d8c34: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 6721356a-6e94-43f1-8433-7629cc3d8c34 + jinja: '" ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + 725bb88d-9eb5-493f-9114-c0cbef6418ad: !Template + answer_choices: A ||| B + id: 725bb88d-9eb5-493f-9114-c0cbef6418ad jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -109,26 +135,26 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: better_A_B reference: '' - 8867366b-9956-4d75-a3fc-1e1b84723e22: !Template - answer_choices: 1 ||| 2 - id: 8867366b-9956-4d75-a3fc-1e1b84723e22 + a6928a46-a55f-4190-b24b-d6f8c40dd5af: !Template + answer_choices: A ||| B + id: a6928a46-a55f-4190-b24b-d6f8c40dd5af jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -148,31 +174,32 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: grammatical_between_A_B reference: '' - 8936931d-af83-498b-b4e6-1a1551f03206: !Template - answer_choices: A ||| B - id: 8936931d-af83-498b-b4e6-1a1551f03206 + c0b7e59d-d702-4e53-9de2-a4e1e8916634: !Template + answer_choices: 1 ||| 2 + id: c0b7e59d-d702-4e53-9de2-a4e1e8916634 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -186,11 +213,22 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: grammatical_which_one_1_2 + reference: '' + d673a952-18e4-48f2-a64e-6cad36affa62: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: d673a952-18e4-48f2-a64e-6cad36affa62 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: true_null_prompt reference: '' - 9b4dc3b2-723e-4b84-a8f9-f6e3794c9f0c: !Template + d6f53c48-9d17-440f-bc3d-6d01005d2ab4: !Template answer_choices: A ||| B - id: 9b4dc3b2-723e-4b84-a8f9-f6e3794c9f0c + id: d6f53c48-9d17-440f-bc3d-6d01005d2ab4 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer A or B. @@ -227,9 +265,9 @@ templates: original_task: true name: grammatical_which_one_A_B reference: '' - b20d7f8e-e71d-4b26-9c56-35e121385e48: !Template + d81c7f05-20d8-459d-8357-4206356e92b9: !Template answer_choices: Yes ||| No - id: b20d7f8e-e71d-4b26-9c56-35e121385e48 + id: d81c7f05-20d8-459d-8357-4206356e92b9 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -243,25 +281,9 @@ templates: original_task: false name: single_sentence_bad_no_yes reference: '' - ee3b718f-6016-4488-b2be-7de1f775d3cb: !Template - answer_choices: Yes ||| No - id: ee3b718f-6016-4488-b2be-7de1f775d3cb - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no - reference: '' - f3542e20-350e-498a-b331-270b8dae4c25: !Template + febf1477-3dc8-4b8a-a024-8f32fd63d084: !Template answer_choices: A ||| B - id: f3542e20-350e-498a-b331-270b8dae4c25 + id: febf1477-3dc8-4b8a-a024-8f32fd63d084 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -280,7 +302,7 @@ templates: {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order + Which one is a better sentence of English, B or A? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -295,5 +317,5 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: better_B_A reference: '' diff --git a/promptsource/templates/blimp/animate_subject_trans/templates.yaml b/promptsource/templates/blimp/animate_subject_trans/templates.yaml index 6c189a8d6..3e2cf7cdb 100644 --- a/promptsource/templates/blimp/animate_subject_trans/templates.yaml +++ b/promptsource/templates/blimp/animate_subject_trans/templates.yaml @@ -1,9 +1,25 @@ dataset: blimp subset: animate_subject_trans templates: - 32855303-3299-4dc0-98ba-db9c954c8deb: !Template + 064c8d03-21f3-4c53-8b38-13068809707a: !Template answer_choices: Yes ||| No - id: 32855303-3299-4dc0-98ba-db9c954c8deb + id: 064c8d03-21f3-4c53-8b38-13068809707a + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes + reference: '' + 1c3fba07-ae44-4318-a279-79b08df8e017: !Template + answer_choices: Yes ||| No + id: 1c3fba07-ae44-4318-a279-79b08df8e017 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -17,62 +33,51 @@ templates: original_task: false name: single_sentence_bad_yes_no reference: '' - 48f85408-43aa-4051-9811-7b88a59da2b1: !Template - answer_choices: A ||| B - id: 48f85408-43aa-4051-9811-7b88a59da2b1 - jinja: '{% set shuffled_order = [0, 1] | random %} - - Read the following two sentences. - - {% if shuffled_order == 0 %} - - Sentence A: {{ sentence_good }} - - Sentence B: {{ sentence_bad }} - - {% else %} - - Sentence A: {{ sentence_bad }} - - Sentence B: {{ sentence_good }} - - {% endif %} - - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} + 1ddd4097-a296-4d71-aaab-906b57edafe9: !Template + answer_choices: Yes ||| No + id: 1ddd4097-a296-4d71-aaab-906b57edafe9 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - {% endif %}' + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy + original_task: false + name: single_sentence_good_no_yes + reference: '' + 2adf5f8f-8bd0-4ca1-b916-decedf6edc46: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 2adf5f8f-8bd0-4ca1-b916-decedf6edc46 + jinja: '" ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy original_task: true - name: better_B_A + name: null_prompt reference: '' - 58930bb5-668c-4a7f-b54f-c0bf97cb1eb6: !Template - answer_choices: A ||| B - id: 58930bb5-668c-4a7f-b54f-c0bf97cb1eb6 + 2b6b48d3-cb8e-4ea4-ad31-3285e40c7a03: !Template + answer_choices: 1 ||| 2 + id: 2b6b48d3-cb8e-4ea4-ad31-3285e40c7a03 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -92,64 +97,31 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B - reference: '' - 664030cf-5b46-48a2-afbb-83e6301574bf: !Template - answer_choices: Yes ||| No - id: 664030cf-5b46-48a2-afbb-83e6301574bf - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes - reference: '' - 6a607ce0-231e-4634-9430-7a734bf8c3a7: !Template - answer_choices: Yes ||| No - id: 6a607ce0-231e-4634-9430-7a734bf8c3a7 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no + name: grammatical_between_1_2 reference: '' - b20918e6-936e-4fd2-afc2-c0a2324431ed: !Template + 2e8da105-49ba-4730-b0fa-1736d634d8d8: !Template answer_choices: A ||| B - id: b20918e6-936e-4fd2-afc2-c0a2324431ed + id: 2e8da105-49ba-4730-b0fa-1736d634d8d8 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -163,11 +135,11 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: better_A_B reference: '' - ca52ffb8-898f-48d7-a9eb-8fc015fcf4e9: !Template + 5c1a565e-a0d4-408f-b215-8544a754894d: !Template answer_choices: A ||| B - id: ca52ffb8-898f-48d7-a9eb-8fc015fcf4e9 + id: 5c1a565e-a0d4-408f-b215-8544a754894d jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -186,7 +158,7 @@ templates: {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order + Which one is a better sentence of English, B or A? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -201,11 +173,11 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: better_B_A reference: '' - cada8bfd-4457-4e59-94a2-1c30878943fe: !Template + 73297481-c7b1-4998-8496-7fcdcc45b98c: !Template answer_choices: 1 ||| 2 - id: cada8bfd-4457-4e59-94a2-1c30878943fe + id: 73297481-c7b1-4998-8496-7fcdcc45b98c jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer 1 or 2. @@ -242,40 +214,51 @@ templates: original_task: true name: grammatical_which_one_1_2 reference: '' - d853c50f-16e5-4cc0-9af6-13bbf6264a27: !Template + 7903aba0-972f-4568-bed5-9318708b860f: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 7903aba0-972f-4568-bed5-9318708b860f + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: true_null_prompt + reference: '' + 9171e40d-c652-43c7-9048-b62f2f64fe35: !Template answer_choices: Yes ||| No - id: d853c50f-16e5-4cc0-9af6-13bbf6264a27 + id: 9171e40d-c652-43c7-9048-b62f2f64fe35 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_bad_no_yes + name: single_sentence_good_yes_no reference: '' - e559f4b3-d385-42e4-8350-5f154c918242: !Template - answer_choices: 1 ||| 2 - id: e559f4b3-d385-42e4-8350-5f154c918242 + 9b16b800-abea-4b12-91a9-361c6ab05ce6: !Template + answer_choices: A ||| B + id: 9b16b800-abea-4b12-91a9-361c6ab05ce6 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -295,5 +278,44 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: grammatical_between_A_B + reference: '' + f874c53d-e05d-4ee2-81cf-b01968e4033f: !Template + answer_choices: A ||| B + id: f874c53d-e05d-4ee2-81cf-b01968e4033f + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_A_B reference: '' diff --git a/promptsource/templates/blimp/causative/templates.yaml b/promptsource/templates/blimp/causative/templates.yaml index 945285318..3107eb646 100644 --- a/promptsource/templates/blimp/causative/templates.yaml +++ b/promptsource/templates/blimp/causative/templates.yaml @@ -1,12 +1,28 @@ dataset: blimp subset: causative templates: - 012dd349-6fcc-4000-8f3b-3c15875a370e: !Template + 168f4a04-eabd-44f5-80b6-9599b90f0911: !Template + answer_choices: Yes ||| No + id: 168f4a04-eabd-44f5-80b6-9599b90f0911 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes + reference: '' + 17de8f10-c62f-42d7-a800-fb34e70a9c3a: !Template answer_choices: 1 ||| 2 - id: 012dd349-6fcc-4000-8f3b-3c15875a370e + id: 17de8f10-c62f-42d7-a800-fb34e70a9c3a jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -38,47 +54,32 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 - reference: '' - 09bce0c9-f733-41fe-b15d-c6984aef3e1f: !Template - answer_choices: Yes ||| No - id: 09bce0c9-f733-41fe-b15d-c6984aef3e1f - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes + name: grammatical_between_1_2 reference: '' - 232187ef-b6c2-44f2-affd-1aa152265d05: !Template + 291a8522-60ac-4634-8fb9-5fbbf25154f2: !Template answer_choices: A ||| B - id: 232187ef-b6c2-44f2-affd-1aa152265d05 + id: 291a8522-60ac-4634-8fb9-5fbbf25154f2 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -92,11 +93,22 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: grammatical_which_one_A_B reference: '' - 4479276c-a0ec-4b7a-bb9e-a1bc23a9bd3a: !Template + 29d6a88c-b38d-4763-b83a-cb5e668ada75: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 29d6a88c-b38d-4763-b83a-cb5e668ada75 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: true_null_prompt + reference: '' + 749bf8fc-7aba-4a96-a080-3bc2498c3589: !Template answer_choices: A ||| B - id: 4479276c-a0ec-4b7a-bb9e-a1bc23a9bd3a + id: 749bf8fc-7aba-4a96-a080-3bc2498c3589 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -115,7 +127,7 @@ templates: {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order + Which one is a better sentence of English, A or B? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -130,26 +142,69 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: better_A_B reference: '' - 78349830-65f8-4ec3-99c5-9b6c01b38358: !Template - answer_choices: 1 ||| 2 - id: 78349830-65f8-4ec3-99c5-9b6c01b38358 + 8249180d-bd45-4e40-8fe9-af9c44faeaa9: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 8249180d-bd45-4e40-8fe9-af9c44faeaa9 + jinja: '" ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + 9fad506f-59eb-41b9-823e-8840b1051d34: !Template + answer_choices: Yes ||| No + id: 9fad506f-59eb-41b9-823e-8840b1051d34 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + c0e592e1-f655-4246-87b0-a63a66d90609: !Template + answer_choices: Yes ||| No + id: c0e592e1-f655-4246-87b0-a63a66d90609 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes + reference: '' + e4d278e5-1aa8-44e4-a56a-92261dedd5e9: !Template + answer_choices: A ||| B + id: e4d278e5-1aa8-44e4-a56a-92261dedd5e9 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -169,32 +224,31 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: grammatical_between_A_B reference: '' - 8fa1c4e1-1e82-47ad-abe7-5dc4c0d7c2ce: !Template + e8f0e6c3-43e4-4351-b590-482b6651173e: !Template answer_choices: A ||| B - id: 8fa1c4e1-1e82-47ad-abe7-5dc4c0d7c2ce + id: e8f0e6c3-43e4-4351-b590-482b6651173e jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -208,26 +262,26 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: better_B_A reference: '' - 9c7358b2-4a6b-4575-a4a7-74f6396e4ff2: !Template - answer_choices: A ||| B - id: 9c7358b2-4a6b-4575-a4a7-74f6396e4ff2 + ef9fd9f5-1656-4251-80d2-5f79e91211fc: !Template + answer_choices: 1 ||| 2 + id: ef9fd9f5-1656-4251-80d2-5f79e91211fc jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -247,27 +301,11 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B - reference: '' - b0fdd869-801e-4455-aeb7-19b145c11833: !Template - answer_choices: Yes ||| No - id: b0fdd869-801e-4455-aeb7-19b145c11833 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes + name: grammatical_which_one_1_2 reference: '' - d9a76e7b-438a-4780-8045-50db5b00e9fb: !Template + f14fbe66-2bfb-4863-bb97-6c8aba7c290a: !Template answer_choices: Yes ||| No - id: d9a76e7b-438a-4780-8045-50db5b00e9fb + id: f14fbe66-2bfb-4863-bb97-6c8aba7c290a jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -281,19 +319,3 @@ templates: original_task: false name: single_sentence_good_yes_no reference: '' - ec52732e-092c-48c9-8b6d-99e205f5ab8e: !Template - answer_choices: Yes ||| No - id: ec52732e-092c-48c9-8b6d-99e205f5ab8e - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no - reference: '' diff --git a/promptsource/templates/blimp/complex_NP_island/templates.yaml b/promptsource/templates/blimp/complex_NP_island/templates.yaml index 6482173d0..40b4ea0ae 100644 --- a/promptsource/templates/blimp/complex_NP_island/templates.yaml +++ b/promptsource/templates/blimp/complex_NP_island/templates.yaml @@ -1,41 +1,59 @@ dataset: blimp subset: complex_NP_island templates: - 1de5ac17-e081-4c30-af42-be78b46550c5: !Template - answer_choices: Yes ||| No - id: 1de5ac17-e081-4c30-af42-be78b46550c5 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + 10dfe420-443f-49aa-9022-af2dd3d270bb: !Template + answer_choices: 1 ||| 2 + id: 10dfe420-443f-49aa-9022-af2dd3d270bb + jinja: '{% set shuffled_order = [0, 1] | random %} - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: false - name: single_sentence_bad_no_yes + original_task: true + name: grammatical_between_1_2 reference: '' - 49c874fd-6627-4788-aa79-e7416f3c8fbf: !Template - answer_choices: Yes ||| No - id: 49c874fd-6627-4788-aa79-e7416f3c8fbf - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + 2699352c-8851-44a5-9e53-2e1f7d047fdd: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 2699352c-8851-44a5-9e53-2e1f7d047fdd + jinja: ' ||| {{ sentence_good }}' metadata: !TemplateMetadata - choices_in_prompt: true + choices_in_prompt: false metrics: - Accuracy - original_task: false - name: single_sentence_good_yes_no + original_task: true + name: true_null_prompt reference: '' - 740e97d4-82c2-4502-a783-97639755143d: !Template + 44ec48a9-bf3a-4a40-9482-e22342b17ef5: !Template answer_choices: A ||| B - id: 740e97d4-82c2-4502-a783-97639755143d + id: 44ec48a9-bf3a-4a40-9482-e22342b17ef5 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -71,24 +89,24 @@ templates: original_task: true name: better_B_A reference: '' - 950ea90c-c8d6-485a-916f-4f684c34ea89: !Template - answer_choices: 1 ||| 2 - id: 950ea90c-c8d6-485a-916f-4f684c34ea89 + 5d8020db-d93c-4d65-806e-1f6974e23393: !Template + answer_choices: A ||| B + id: 5d8020db-d93c-4d65-806e-1f6974e23393 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -108,11 +126,27 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: grammatical_between_A_B + reference: '' + 642a5a47-bd13-4c2a-8948-a371520549fb: !Template + answer_choices: Yes ||| No + id: 642a5a47-bd13-4c2a-8948-a371520549fb + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no reference: '' - a3fd445c-90fd-462b-9965-7a32b069b792: !Template + 75916a91-d763-43dc-8e0f-b43efd72e2e4: !Template answer_choices: A ||| B - id: a3fd445c-90fd-462b-9965-7a32b069b792 + id: 75916a91-d763-43dc-8e0f-b43efd72e2e4 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -148,9 +182,9 @@ templates: original_task: true name: better_A_B reference: '' - c5f50d0a-e669-4061-85c9-d39e6e654d3c: !Template + 838919e3-8e61-4843-8469-dfe7ddab9027: !Template answer_choices: A ||| B - id: c5f50d0a-e669-4061-85c9-d39e6e654d3c + id: 838919e3-8e61-4843-8469-dfe7ddab9027 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer A or B. @@ -187,40 +221,51 @@ templates: original_task: true name: grammatical_which_one_A_B reference: '' - cb4b4bec-0925-445e-b0d0-cc100a67016b: !Template + 986f3bef-10b5-47b2-873c-950f3e5f1d8e: !Template answer_choices: Yes ||| No - id: cb4b4bec-0925-445e-b0d0-cc100a67016b + id: 986f3bef-10b5-47b2-873c-950f3e5f1d8e jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_good_no_yes + name: single_sentence_bad_no_yes reference: '' - cdc0ba7d-d40b-4312-91e3-56638b2c3ba4: !Template - answer_choices: A ||| B - id: cdc0ba7d-d40b-4312-91e3-56638b2c3ba4 + 9d32b8a0-58eb-4f5d-97d8-32bc5e69484f: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 9d32b8a0-58eb-4f5d-97d8-32bc5e69484f + jinja: '" ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + 9f96111f-2282-4c40-9110-b0dadc97297b: !Template + answer_choices: 1 ||| 2 + id: 9f96111f-2282-4c40-9110-b0dadc97297b jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -240,60 +285,37 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: grammatical_which_one_1_2 reference: '' - d082b242-cbb5-4e23-848c-479b34275ac3: !Template + ca8ab9ef-b864-41a5-acc7-4ae8064c6583: !Template answer_choices: Yes ||| No - id: d082b242-cbb5-4e23-848c-479b34275ac3 + id: ca8ab9ef-b864-41a5-acc7-4ae8064c6583 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_bad_yes_no + name: single_sentence_good_yes_no reference: '' - ed420a31-5808-4a93-8e1a-5b06e953973c: !Template - answer_choices: 1 ||| 2 - id: ed420a31-5808-4a93-8e1a-5b06e953973c - jinja: '{% set shuffled_order = [0, 1] | random %} - - Which one of the following sentences is grammatical? Please answer 1 or 2. - - {% if shuffled_order == 0 %} - - 1: {{ sentence_good }} - - 2: {{ sentence_bad }} - - {% else %} - - 1: {{ sentence_bad }} - - 2: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} + e7f0c3da-177c-492c-aa3f-729e7059ef21: !Template + answer_choices: Yes ||| No + id: e7f0c3da-177c-492c-aa3f-729e7059ef21 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - {% endif %}' + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: grammatical_which_one_1_2 + original_task: false + name: single_sentence_good_no_yes reference: '' diff --git a/promptsource/templates/blimp/coordinate_structure_constraint_complex_left_branch/templates.yaml b/promptsource/templates/blimp/coordinate_structure_constraint_complex_left_branch/templates.yaml index 2469f95f1..2bc5b7cd4 100644 --- a/promptsource/templates/blimp/coordinate_structure_constraint_complex_left_branch/templates.yaml +++ b/promptsource/templates/blimp/coordinate_structure_constraint_complex_left_branch/templates.yaml @@ -1,56 +1,24 @@ dataset: blimp subset: coordinate_structure_constraint_complex_left_branch templates: - 24075d3e-4262-4582-8b8a-0a4e7e74178c: !Template - answer_choices: Yes ||| No - id: 24075d3e-4262-4582-8b8a-0a4e7e74178c - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no - reference: '' - 24496016-429e-444a-8b01-9ec42e37af02: !Template - answer_choices: Yes ||| No - id: 24496016-429e-444a-8b01-9ec42e37af02 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes - reference: '' - 412f33ac-7425-421c-aeef-cb3701bca420: !Template - answer_choices: A ||| B - id: 412f33ac-7425-421c-aeef-cb3701bca420 + 1dc0a548-237a-4bfd-9c72-dbfab9344488: !Template + answer_choices: 1 ||| 2 + id: 1dc0a548-237a-4bfd-9c72-dbfab9344488 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -70,49 +38,11 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B - reference: '' - 709eb4f6-8a40-4012-99f0-3054dde7b11b: !Template - answer_choices: A ||| B - id: 709eb4f6-8a40-4012-99f0-3054dde7b11b - jinja: '{% set shuffled_order = [0, 1] | random %} - - Read the following two sentences. - - {% if shuffled_order == 0 %} - - Sentence A: {{ sentence_good }} - - Sentence B: {{ sentence_bad }} - - {% else %} - - Sentence A: {{ sentence_bad }} - - Sentence B: {{ sentence_good }} - - {% endif %} - - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} - - {% endif %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: better_A_B + name: grammatical_between_1_2 reference: '' - 960c314c-f5ee-4350-b301-8a288b335458: !Template + 266fdc02-5341-4e31-b5ae-879e7954e6ae: !Template answer_choices: A ||| B - id: 960c314c-f5ee-4350-b301-8a288b335458 + id: 266fdc02-5341-4e31-b5ae-879e7954e6ae jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer A or B. @@ -149,9 +79,57 @@ templates: original_task: true name: grammatical_which_one_A_B reference: '' - c1fd0ab7-e80b-4149-b198-5e4d07cf014f: !Template + 2be386e3-f37a-42fa-8d87-322ef523dfb8: !Template + answer_choices: Yes ||| No + id: 2be386e3-f37a-42fa-8d87-322ef523dfb8 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 5bd12502-5a68-47d9-871c-53c35d202dbb: !Template + answer_choices: Yes ||| No + id: 5bd12502-5a68-47d9-871c-53c35d202dbb + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes + reference: '' + 5df8f7b5-014d-4478-b300-355f71a1b029: !Template answer_choices: Yes ||| No - id: c1fd0ab7-e80b-4149-b198-5e4d07cf014f + id: 5df8f7b5-014d-4478-b300-355f71a1b029 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 7baef85f-4114-4d4d-9f2a-90d38675d5aa: !Template + answer_choices: Yes ||| No + id: 7baef85f-4114-4d4d-9f2a-90d38675d5aa jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -165,9 +143,9 @@ templates: original_task: false name: single_sentence_bad_no_yes reference: '' - c68f61f6-b53b-457c-99a3-5802157a9194: !Template + 89c9a342-28a5-4d6e-935a-03c08c02cfbf: !Template answer_choices: A ||| B - id: c68f61f6-b53b-457c-99a3-5802157a9194 + id: 89c9a342-28a5-4d6e-935a-03c08c02cfbf jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -186,7 +164,7 @@ templates: {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order + Which one is a better sentence of English, A or B? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -201,14 +179,14 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: better_A_B reference: '' - c7fa415e-9375-44e4-9523-e81fe999ed7d: !Template + 8efe286f-814b-42c0-925d-6ddc7c7cb607: !Template answer_choices: 1 ||| 2 - id: c7fa415e-9375-44e4-9523-e81fe999ed7d + id: 8efe286f-814b-42c0-925d-6ddc7c7cb607 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -240,42 +218,48 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: grammatical_which_one_1_2 reference: '' - e2a36515-933c-435f-ace9-f664b0654717: !Template - answer_choices: Yes ||| No - id: e2a36515-933c-435f-ace9-f664b0654717 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' + b8c2ed75-8b30-4962-b02d-1638413c8d14: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: b8c2ed75-8b30-4962-b02d-1638413c8d14 + jinja: '" ||| {{ sentence_good }}' metadata: !TemplateMetadata - choices_in_prompt: true + choices_in_prompt: false metrics: - Accuracy - original_task: false - name: single_sentence_bad_yes_no + original_task: true + name: null_prompt reference: '' - eae2f6d9-6105-4d5f-900c-d4faf2a68474: !Template - answer_choices: 1 ||| 2 - id: eae2f6d9-6105-4d5f-900c-d4faf2a68474 + bd7b9332-c8ac-4626-9484-b94644bd4569: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: bd7b9332-c8ac-4626-9484-b94644bd4569 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: true_null_prompt + reference: '' + c4045c0d-0dd3-4370-96a9-0d5da4b92ebb: !Template + answer_choices: A ||| B + id: c4045c0d-0dd3-4370-96a9-0d5da4b92ebb jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -295,5 +279,43 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: grammatical_between_A_B + reference: '' + e21c0a63-bfaa-4b6c-8be6-7cd2ca9e7a7f: !Template + answer_choices: A ||| B + id: e21c0a63-bfaa-4b6c-8be6-7cd2ca9e7a7f + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: better_B_A reference: '' diff --git a/promptsource/templates/blimp/coordinate_structure_constraint_object_extraction/templates.yaml b/promptsource/templates/blimp/coordinate_structure_constraint_object_extraction/templates.yaml index c60e8df62..b68636d7e 100644 --- a/promptsource/templates/blimp/coordinate_structure_constraint_object_extraction/templates.yaml +++ b/promptsource/templates/blimp/coordinate_structure_constraint_object_extraction/templates.yaml @@ -1,47 +1,31 @@ dataset: blimp subset: coordinate_structure_constraint_object_extraction templates: - 027b55c3-2903-4613-a898-33185f9f51b0: !Template - answer_choices: A ||| B - id: 027b55c3-2903-4613-a898-33185f9f51b0 - jinja: '{% set shuffled_order = [0, 1] | random %} - - Read the following two sentences. - - {% if shuffled_order == 0 %} - - Sentence A: {{ sentence_good }} - - Sentence B: {{ sentence_bad }} - - {% else %} - - Sentence A: {{ sentence_bad }} - - Sentence B: {{ sentence_good }} - - {% endif %} - - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} - - {% endif %}' + 1c920ce5-a4e1-4429-ad69-cb75b85c2ee5: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 1c920ce5-a4e1-4429-ad69-cb75b85c2ee5 + jinja: '" ||| {{ sentence_good }}' metadata: !TemplateMetadata - choices_in_prompt: true + choices_in_prompt: false metrics: - Accuracy original_task: true - name: better_A_B + name: null_prompt reference: '' - 0297743c-253c-4b9c-bb56-d5248aa64fe9: !Template + 1d6280db-3f10-476d-b64d-a6975f00717c: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 1d6280db-3f10-476d-b64d-a6975f00717c + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: true_null_prompt + reference: '' + 26d3df71-b001-41f1-b3c4-a8f94ecc8bc8: !Template answer_choices: Yes ||| No - id: 0297743c-253c-4b9c-bb56-d5248aa64fe9 + id: 26d3df71-b001-41f1-b3c4-a8f94ecc8bc8 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -55,29 +39,30 @@ templates: original_task: false name: single_sentence_good_yes_no reference: '' - 0d28a4cf-03eb-4ce1-af7a-ff90e63d13bb: !Template - answer_choices: A ||| B - id: 0d28a4cf-03eb-4ce1-af7a-ff90e63d13bb + 288a3c31-0441-41f5-bead-d1e5ebb8a931: !Template + answer_choices: 1 ||| 2 + id: 288a3c31-0441-41f5-bead-d1e5ebb8a931 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -91,11 +76,11 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: grammatical_which_one_1_2 reference: '' - 5b2491ae-f896-41f6-9712-6e44f4f4f8ac: !Template + 4178601f-7e50-4fd1-9c2f-c9eebe0ce75c: !Template answer_choices: 1 ||| 2 - id: 5b2491ae-f896-41f6-9712-6e44f4f4f8ac + id: 4178601f-7e50-4fd1-9c2f-c9eebe0ce75c jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -132,56 +117,40 @@ templates: original_task: true name: grammatical_between_1_2 reference: '' - 5f3ef743-71af-4dd4-8f50-abc340a09a48: !Template - answer_choices: Yes ||| No - id: 5f3ef743-71af-4dd4-8f50-abc340a09a48 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes - reference: '' - 6bc44dd4-2150-46f0-abb0-424d00679fd4: !Template + 5b98d834-0256-41ec-9aae-e761013fa8fd: !Template answer_choices: Yes ||| No - id: 6bc44dd4-2150-46f0-abb0-424d00679fd4 + id: 5b98d834-0256-41ec-9aae-e761013fa8fd jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_bad_yes_no + name: single_sentence_good_no_yes reference: '' - 7f56b76f-9615-47a1-83d3-9c19e98481e5: !Template - answer_choices: 1 ||| 2 - id: 7f56b76f-9615-47a1-83d3-9c19e98481e5 + 84105f1d-256a-4c3f-8cd8-eb26e8393283: !Template + answer_choices: A ||| B + id: 84105f1d-256a-4c3f-8cd8-eb26e8393283 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -201,32 +170,31 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: grammatical_between_A_B reference: '' - 92380e34-cddb-45ca-b8e6-a68185aa2728: !Template + 8c227511-2cbb-4b9d-83c5-cd5f7143f123: !Template answer_choices: A ||| B - id: 92380e34-cddb-45ca-b8e6-a68185aa2728 + id: 8c227511-2cbb-4b9d-83c5-cd5f7143f123 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -240,27 +208,43 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: better_B_A reference: '' - a4d051ee-6bda-4def-80a8-021d696dee04: !Template + 8f521740-cd3b-414e-9a14-d9b832ecd09d: !Template answer_choices: Yes ||| No - id: a4d051ee-6bda-4def-80a8-021d696dee04 + id: 8f521740-cd3b-414e-9a14-d9b832ecd09d jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_good_no_yes + name: single_sentence_bad_yes_no reference: '' - b32481b8-c3e1-48c1-9dd2-8582b730a8d8: !Template + 97ca2291-9977-4526-abdf-a4baafea371e: !Template + answer_choices: Yes ||| No + id: 97ca2291-9977-4526-abdf-a4baafea371e + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes + reference: '' + bfb80d33-9748-46e4-bc20-f02ca0ed5681: !Template answer_choices: A ||| B - id: b32481b8-c3e1-48c1-9dd2-8582b730a8d8 + id: bfb80d33-9748-46e4-bc20-f02ca0ed5681 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer A or B. @@ -297,3 +281,41 @@ templates: original_task: true name: grammatical_which_one_A_B reference: '' + cbb7b2e0-9a13-449d-ad23-2fb3247dc78d: !Template + answer_choices: A ||| B + id: cbb7b2e0-9a13-449d-ad23-2fb3247dc78d + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: better_A_B + reference: '' diff --git a/promptsource/templates/blimp/determiner_noun_agreement_1/templates.yaml b/promptsource/templates/blimp/determiner_noun_agreement_1/templates.yaml index 69733dcc2..66a848ff2 100644 --- a/promptsource/templates/blimp/determiner_noun_agreement_1/templates.yaml +++ b/promptsource/templates/blimp/determiner_noun_agreement_1/templates.yaml @@ -1,30 +1,61 @@ dataset: blimp subset: determiner_noun_agreement_1 templates: - 2c33dbe3-d023-4ed2-94f1-ad3100089999: !Template - answer_choices: 1 ||| 2 - id: 2c33dbe3-d023-4ed2-94f1-ad3100089999 + 1db52d9a-70f0-44c8-a6e6-77fbfa6a9a18: !Template + answer_choices: Yes ||| No + id: 1db52d9a-70f0-44c8-a6e6-77fbfa6a9a18 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes + reference: '' + 29da8721-7fb9-44e5-8b47-5aaba62bfc4b: !Template + answer_choices: Yes ||| No + id: 29da8721-7fb9-44e5-8b47-5aaba62bfc4b + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 45aa0402-d965-48e9-8cd5-a44f3176172f: !Template + answer_choices: A ||| B + id: 45aa0402-d965-48e9-8cd5-a44f3176172f jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -38,14 +69,25 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: better_B_A reference: '' - 33b939ff-3a68-4696-94b9-624c5bb99196: !Template + 4a2b1c8f-a224-49b8-9ed7-af5b74ef6a7f: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 4a2b1c8f-a224-49b8-9ed7-af5b74ef6a7f + jinja: '" ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + 55ad36f6-af14-4cdc-80fa-661aad9b38b3: !Template answer_choices: 1 ||| 2 - id: 33b939ff-3a68-4696-94b9-624c5bb99196 + id: 55ad36f6-af14-4cdc-80fa-661aad9b38b3 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -77,43 +119,11 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 - reference: '' - 534ca5a0-f7d7-44dd-bd5c-b560b8e527dc: !Template - answer_choices: Yes ||| No - id: 534ca5a0-f7d7-44dd-bd5c-b560b8e527dc - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no - reference: '' - 82f50a6d-dfe8-4181-a17a-788051e83668: !Template - answer_choices: Yes ||| No - id: 82f50a6d-dfe8-4181-a17a-788051e83668 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes + name: grammatical_which_one_1_2 reference: '' - 8bcb4d94-4645-4d17-a61f-ef01ce1d3224: !Template + 5c4eb1e0-370b-49cd-8399-fb6296582eee: !Template answer_choices: A ||| B - id: 8bcb4d94-4645-4d17-a61f-ef01ce1d3224 + id: 5c4eb1e0-370b-49cd-8399-fb6296582eee jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer A or B. @@ -150,45 +160,30 @@ templates: original_task: true name: grammatical_between_A_B reference: '' - 90380c6c-6997-489e-ad66-294712be986f: !Template - answer_choices: Yes ||| No - id: 90380c6c-6997-489e-ad66-294712be986f - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes - reference: '' - 9d442c90-74a2-49e4-afb2-083652d89a95: !Template - answer_choices: A ||| B - id: 9d442c90-74a2-49e4-afb2-083652d89a95 + 64321690-5afb-428d-8339-67415356f5f9: !Template + answer_choices: 1 ||| 2 + id: 64321690-5afb-428d-8339-67415356f5f9 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -202,11 +197,27 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: grammatical_between_1_2 + reference: '' + a069a35b-c357-469d-b307-d92a57e3fde7: !Template + answer_choices: Yes ||| No + id: a069a35b-c357-469d-b307-d92a57e3fde7 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no reference: '' - a3170a13-b9eb-445f-bb4a-c202cd9abbec: !Template + b241bf1a-e1ce-4f65-8962-7b176760dce1: !Template answer_choices: A ||| B - id: a3170a13-b9eb-445f-bb4a-c202cd9abbec + id: b241bf1a-e1ce-4f65-8962-7b176760dce1 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -225,7 +236,7 @@ templates: {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order + Which one is a better sentence of English, A or B? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -240,27 +251,11 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A - reference: '' - a7b7f359-b815-4e92-87d8-ab8e3b07121d: !Template - answer_choices: Yes ||| No - id: a7b7f359-b815-4e92-87d8-ab8e3b07121d - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no + name: better_A_B reference: '' - f2023fa2-5bdc-45fa-934e-e517b41fcea5: !Template + b71c1107-873d-4f7b-a55d-2d421504b849: !Template answer_choices: A ||| B - id: f2023fa2-5bdc-45fa-934e-e517b41fcea5 + id: b71c1107-873d-4f7b-a55d-2d421504b849 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer A or B. @@ -297,3 +292,30 @@ templates: original_task: true name: grammatical_which_one_A_B reference: '' + f6fefaec-b1f8-4543-97c4-282a541219f4: !Template + answer_choices: Yes ||| No + id: f6fefaec-b1f8-4543-97c4-282a541219f4 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes + reference: '' + f80aed19-f670-4722-9513-efb0d7c654c7: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: f80aed19-f670-4722-9513-efb0d7c654c7 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: true_null_prompt + reference: '' diff --git a/promptsource/templates/blimp/determiner_noun_agreement_2/templates.yaml b/promptsource/templates/blimp/determiner_noun_agreement_2/templates.yaml index ba1c6b248..cc603166c 100644 --- a/promptsource/templates/blimp/determiner_noun_agreement_2/templates.yaml +++ b/promptsource/templates/blimp/determiner_noun_agreement_2/templates.yaml @@ -1,25 +1,9 @@ dataset: blimp subset: determiner_noun_agreement_2 templates: - 287f554f-0296-4745-854f-55a7f0615c1a: !Template - answer_choices: Yes ||| No - id: 287f554f-0296-4745-854f-55a7f0615c1a - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no - reference: '' - 3ba5ee24-2973-463c-863a-d3d17ea1cbe8: !Template + 124c66e5-f4fe-44a6-9942-0fb38a0b3f5e: !Template answer_choices: A ||| B - id: 3ba5ee24-2973-463c-863a-d3d17ea1cbe8 + id: 124c66e5-f4fe-44a6-9942-0fb38a0b3f5e jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer A or B. @@ -56,24 +40,40 @@ templates: original_task: true name: grammatical_between_A_B reference: '' - 3d19ee7e-f9ce-4eee-8a54-f5175ff2d32b: !Template - answer_choices: A ||| B - id: 3d19ee7e-f9ce-4eee-8a54-f5175ff2d32b + 15d41b02-c038-4b99-9ea3-1b9696dc30e5: !Template + answer_choices: Yes ||| No + id: 15d41b02-c038-4b99-9ea3-1b9696dc30e5 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 22129942-86ae-4290-93b4-2ce9cceb2503: !Template + answer_choices: 1 ||| 2 + id: 22129942-86ae-4290-93b4-2ce9cceb2503 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -93,27 +93,11 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B - reference: '' - 5898404e-1736-4df5-bdc6-e9d40467138d: !Template - answer_choices: Yes ||| No - id: 5898404e-1736-4df5-bdc6-e9d40467138d - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes + name: grammatical_between_1_2 reference: '' - 924bfa4b-e6bd-45fa-af65-8f94ab33ea78: !Template + 31377a83-cca7-433f-a727-ab3e3b1378cb: !Template answer_choices: A ||| B - id: 924bfa4b-e6bd-45fa-af65-8f94ab33ea78 + id: 31377a83-cca7-433f-a727-ab3e3b1378cb jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -132,7 +116,7 @@ templates: {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order + Which one is a better sentence of English, A or B? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -147,14 +131,14 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: better_A_B reference: '' - aea1cc03-f411-4d5b-bba6-140c30fa6c46: !Template + 3d2bfdcc-50a4-4d36-8bce-3dd109386147: !Template answer_choices: 1 ||| 2 - id: aea1cc03-f411-4d5b-bba6-140c30fa6c46 + id: 3d2bfdcc-50a4-4d36-8bce-3dd109386147 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -186,26 +170,69 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: grammatical_which_one_1_2 reference: '' - b64965c6-e672-4e48-adfb-95e75b9ce537: !Template - answer_choices: 1 ||| 2 - id: b64965c6-e672-4e48-adfb-95e75b9ce537 + 4f70018c-1108-4801-b6a1-161c71bce8b3: !Template + answer_choices: Yes ||| No + id: 4f70018c-1108-4801-b6a1-161c71bce8b3 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 66f08583-4be1-4d57-9c01-6e8077610323: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 66f08583-4be1-4d57-9c01-6e8077610323 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: true_null_prompt + reference: '' + 8b6c24e4-5a25-4fbf-b396-d988e4eee38f: !Template + answer_choices: Yes ||| No + id: 8b6c24e4-5a25-4fbf-b396-d988e4eee38f + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes + reference: '' + 9a6ec137-1fb3-4f6d-96e0-18bc5aaa2e34: !Template + answer_choices: A ||| B + id: 9a6ec137-1fb3-4f6d-96e0-18bc5aaa2e34 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -225,11 +252,27 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: grammatical_which_one_A_B reference: '' - d2394fd4-954f-4b34-9b14-680a6b855fb2: !Template + a870aa59-a74c-4522-81e3-48f5d35b07a7: !Template + answer_choices: Yes ||| No + id: a870aa59-a74c-4522-81e3-48f5d35b07a7 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes + reference: '' + ba8b3eda-b858-4d24-b1a2-7e8f6cdac276: !Template answer_choices: A ||| B - id: d2394fd4-954f-4b34-9b14-680a6b855fb2 + id: ba8b3eda-b858-4d24-b1a2-7e8f6cdac276 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -248,7 +291,7 @@ templates: {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order + Which one is a better sentence of English, B or A? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -263,37 +306,16 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B - reference: '' - ed64a5f2-73c8-4c8d-8fe8-f34023bec7dd: !Template - answer_choices: Yes ||| No - id: ed64a5f2-73c8-4c8d-8fe8-f34023bec7dd - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no + name: better_B_A reference: '' - ef8c6db8-d1e0-48ae-8065-3c140ea3bc0e: !Template - answer_choices: Yes ||| No - id: ef8c6db8-d1e0-48ae-8065-3c140ea3bc0e - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' + dd61ea3c-4770-41ba-9be7-e6803a91433d: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: dd61ea3c-4770-41ba-9be7-e6803a91433d + jinja: '" ||| {{ sentence_good }}' metadata: !TemplateMetadata - choices_in_prompt: true + choices_in_prompt: false metrics: - Accuracy - original_task: false - name: single_sentence_bad_no_yes + original_task: true + name: null_prompt reference: '' diff --git a/promptsource/templates/blimp/determiner_noun_agreement_irregular_1/templates.yaml b/promptsource/templates/blimp/determiner_noun_agreement_irregular_1/templates.yaml index d33b925b1..0ac826893 100644 --- a/promptsource/templates/blimp/determiner_noun_agreement_irregular_1/templates.yaml +++ b/promptsource/templates/blimp/determiner_noun_agreement_irregular_1/templates.yaml @@ -1,25 +1,48 @@ dataset: blimp subset: determiner_noun_agreement_irregular_1 templates: - 281b27db-76af-4a85-8f3d-a34d77a43b76: !Template - answer_choices: Yes ||| No - id: 281b27db-76af-4a85-8f3d-a34d77a43b76 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + 2cc9d0e0-59c4-4140-8120-ec667040b627: !Template + answer_choices: 1 ||| 2 + id: 2cc9d0e0-59c4-4140-8120-ec667040b627 + jinja: '{% set shuffled_order = [0, 1] | random %} - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: false - name: single_sentence_good_yes_no + original_task: true + name: grammatical_between_1_2 reference: '' - 331683dc-bdef-4a2a-8f13-b9da7eaab72b: !Template + 2e41c457-200f-47a1-97f6-fd09a0ab83ed: !Template answer_choices: A ||| B - id: 331683dc-bdef-4a2a-8f13-b9da7eaab72b + id: 2e41c457-200f-47a1-97f6-fd09a0ab83ed jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -55,9 +78,52 @@ templates: original_task: true name: better_B_A reference: '' - 3b852bc6-1d1e-44bb-98a1-ec2675ab5eb2: !Template + 9307601d-e073-4bed-be6d-67a9d39f45f3: !Template + answer_choices: Yes ||| No + id: 9307601d-e073-4bed-be6d-67a9d39f45f3 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 9c2a4478-363d-4caf-9a6a-713dd0f3c86e: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 9c2a4478-363d-4caf-9a6a-713dd0f3c86e + jinja: '" ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + c8609ae6-0f99-4c4a-8f48-aaebbbc9c362: !Template + answer_choices: Yes ||| No + id: c8609ae6-0f99-4c4a-8f48-aaebbbc9c362 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes + reference: '' + c8cedf93-50df-490d-9f37-b7e3fa7a874c: !Template answer_choices: A ||| B - id: 3b852bc6-1d1e-44bb-98a1-ec2675ab5eb2 + id: c8cedf93-50df-490d-9f37-b7e3fa7a874c jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer A or B. @@ -94,47 +160,52 @@ templates: original_task: true name: grammatical_which_one_A_B reference: '' - 5757fd44-5409-4fad-90c5-ccea3750ac7e: !Template - answer_choices: A ||| B - id: 5757fd44-5409-4fad-90c5-ccea3750ac7e - jinja: '{% set shuffled_order = [0, 1] | random %} - - Read the following two sentences. - - {% if shuffled_order == 0 %} - - Sentence A: {{ sentence_good }} - - Sentence B: {{ sentence_bad }} - - {% else %} - - Sentence A: {{ sentence_bad }} - - Sentence B: {{ sentence_good }} - - {% endif %} - - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} - - {{ answer_choices[0] }} - - {% else %} + cecd6e5a-8daa-4102-9888-4fcd6e7c649f: !Template + answer_choices: Yes ||| No + id: cecd6e5a-8daa-4102-9888-4fcd6e7c649f + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - {{ answer_choices[1] }} + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + e498feb2-f347-4d28-a557-804241a81edf: !Template + answer_choices: Yes ||| No + id: e498feb2-f347-4d28-a557-804241a81edf + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - {% endif %}' + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy + original_task: false + name: single_sentence_bad_no_yes + reference: '' + e61e6c8a-8231-47e4-8ac0-b989a93f54a9: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: e61e6c8a-8231-47e4-8ac0-b989a93f54a9 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy original_task: true - name: better_A_B + name: true_null_prompt reference: '' - 58a0dba8-1840-476d-a345-f8b182a13c00: !Template + f13211a8-ddd8-4aef-9757-4bb6f35fdd30: !Template answer_choices: 1 ||| 2 - id: 58a0dba8-1840-476d-a345-f8b182a13c00 + id: f13211a8-ddd8-4aef-9757-4bb6f35fdd30 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer 1 or 2. @@ -171,9 +242,9 @@ templates: original_task: true name: grammatical_which_one_1_2 reference: '' - 843a4a4d-823d-4045-bdae-0e58e6817876: !Template + f8f8680b-dd66-4b56-a324-ac516008d355: !Template answer_choices: A ||| B - id: 843a4a4d-823d-4045-bdae-0e58e6817876 + id: f8f8680b-dd66-4b56-a324-ac516008d355 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer A or B. @@ -210,78 +281,29 @@ templates: original_task: true name: grammatical_between_A_B reference: '' - 882264b2-255b-4796-bacd-8f9f1130cdc1: !Template - answer_choices: Yes ||| No - id: 882264b2-255b-4796-bacd-8f9f1130cdc1 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes - reference: '' - 93ced979-9e52-46b5-b328-d3729b2a541a: !Template - answer_choices: Yes ||| No - id: 93ced979-9e52-46b5-b328-d3729b2a541a - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no - reference: '' - ab3ae97a-002c-4f3b-a9f1-5a2e9413ed8a: !Template - answer_choices: Yes ||| No - id: ab3ae97a-002c-4f3b-a9f1-5a2e9413ed8a - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes - reference: '' - f065fb5a-2bda-4690-b52b-b6abed4c525d: !Template - answer_choices: 1 ||| 2 - id: f065fb5a-2bda-4690-b52b-b6abed4c525d + fd765908-f30f-46aa-bf93-e2ed4473294b: !Template + answer_choices: A ||| B + id: fd765908-f30f-46aa-bf93-e2ed4473294b jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -295,5 +317,5 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: better_A_B reference: '' diff --git a/promptsource/templates/blimp/determiner_noun_agreement_irregular_2/templates.yaml b/promptsource/templates/blimp/determiner_noun_agreement_irregular_2/templates.yaml index 3f4afcb02..e15bdf756 100644 --- a/promptsource/templates/blimp/determiner_noun_agreement_irregular_2/templates.yaml +++ b/promptsource/templates/blimp/determiner_noun_agreement_irregular_2/templates.yaml @@ -1,45 +1,57 @@ dataset: blimp subset: determiner_noun_agreement_irregular_2 templates: - 295e6c73-26c0-46f1-89ea-fb3f4555c659: !Template + 06923da0-136b-4bf9-9dc8-d06779e90f0d: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 06923da0-136b-4bf9-9dc8-d06779e90f0d + jinja: '" ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + 105467bc-f5bc-41fe-99f8-2ac18bcebc7e: !Template answer_choices: Yes ||| No - id: 295e6c73-26c0-46f1-89ea-fb3f4555c659 + id: 105467bc-f5bc-41fe-99f8-2ac18bcebc7e jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_good_yes_no + name: single_sentence_bad_yes_no reference: '' - 36ec8691-698f-4905-8cef-b848e3bd9cd5: !Template + 15f33e2a-3e6b-4efe-b71f-89656d3d8f18: !Template answer_choices: A ||| B - id: 36ec8691-698f-4905-8cef-b848e3bd9cd5 + id: 15f33e2a-3e6b-4efe-b71f-89656d3d8f18 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -53,32 +65,31 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: grammatical_between_A_B reference: '' - 53d4645d-34b7-487c-b95d-ce88d37dd9fc: !Template - answer_choices: 1 ||| 2 - id: 53d4645d-34b7-487c-b95d-ce88d37dd9fc + 25c379da-8e98-453e-bfa6-858c0b06c372: !Template + answer_choices: A ||| B + id: 25c379da-8e98-453e-bfa6-858c0b06c372 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -92,58 +103,26 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 - reference: '' - 5ead7bd5-831c-48aa-871f-1d7c13b6548e: !Template - answer_choices: Yes ||| No - id: 5ead7bd5-831c-48aa-871f-1d7c13b6548e - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no - reference: '' - 6341fd3c-2768-40d2-b9fb-6f69d2baf250: !Template - answer_choices: Yes ||| No - id: 6341fd3c-2768-40d2-b9fb-6f69d2baf250 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes + name: better_A_B reference: '' - 6fd90f4e-72cb-4cc4-8d3c-4c18b8a30910: !Template - answer_choices: A ||| B - id: 6fd90f4e-72cb-4cc4-8d3c-4c18b8a30910 + 7193d72d-63c4-47ce-8722-a3cd69f3d877: !Template + answer_choices: 1 ||| 2 + id: 7193d72d-63c4-47ce-8722-a3cd69f3d877 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -163,31 +142,43 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: grammatical_which_one_1_2 reference: '' - 92ba313f-cefd-4a6f-8449-314bd4dfcbbe: !Template - answer_choices: A ||| B - id: 92ba313f-cefd-4a6f-8449-314bd4dfcbbe + 76d252bc-44ff-40a8-9b09-bf080461385f: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 76d252bc-44ff-40a8-9b09-bf080461385f + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: true_null_prompt + reference: '' + 8aa80b8c-859f-4a84-9967-83a6e54a30fd: !Template + answer_choices: 1 ||| 2 + id: 8aa80b8c-859f-4a84-9967-83a6e54a30fd jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -201,11 +192,11 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: grammatical_between_1_2 reference: '' - 935f5b0c-35bb-41e7-8b7f-f7a632963779: !Template + 9d317cfa-cce0-4774-b327-b435838947a1: !Template answer_choices: Yes ||| No - id: 935f5b0c-35bb-41e7-8b7f-f7a632963779 + id: 9d317cfa-cce0-4774-b327-b435838947a1 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -219,9 +210,9 @@ templates: original_task: false name: single_sentence_good_no_yes reference: '' - c1413ff7-c645-4009-8c9f-f8d62e1c6ca6: !Template + d1773c16-0d07-451e-b327-915929bc0120: !Template answer_choices: A ||| B - id: c1413ff7-c645-4009-8c9f-f8d62e1c6ca6 + id: d1773c16-0d07-451e-b327-915929bc0120 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer A or B. @@ -258,30 +249,29 @@ templates: original_task: true name: grammatical_which_one_A_B reference: '' - c9de7a9e-9c14-4071-a5f3-8899f7fe5702: !Template - answer_choices: 1 ||| 2 - id: c9de7a9e-9c14-4071-a5f3-8899f7fe5702 + db127d8a-f404-402a-b420-ba9519af2707: !Template + answer_choices: A ||| B + id: db127d8a-f404-402a-b420-ba9519af2707 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -295,5 +285,37 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: better_B_A + reference: '' + dbf3fd44-687f-4549-9737-72505fe0067a: !Template + answer_choices: Yes ||| No + id: dbf3fd44-687f-4549-9737-72505fe0067a + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + e9ecf4cc-8541-4104-b61e-0be797b9a2f8: !Template + answer_choices: Yes ||| No + id: e9ecf4cc-8541-4104-b61e-0be797b9a2f8 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes reference: '' diff --git a/promptsource/templates/blimp/determiner_noun_agreement_with_adj_2/templates.yaml b/promptsource/templates/blimp/determiner_noun_agreement_with_adj_2/templates.yaml index 1ce1d6f2c..63a4b9181 100644 --- a/promptsource/templates/blimp/determiner_noun_agreement_with_adj_2/templates.yaml +++ b/promptsource/templates/blimp/determiner_noun_agreement_with_adj_2/templates.yaml @@ -1,13 +1,13 @@ dataset: blimp subset: determiner_noun_agreement_with_adj_2 templates: - 1cc44f9b-dc96-4231-8a7c-c57ff1a124dc: !Template + 501487b3-c60b-4164-a3d4-bc19899b570a: !Template answer_choices: Yes ||| No - id: 1cc44f9b-dc96-4231-8a7c-c57ff1a124dc + id: 501487b3-c60b-4164-a3d4-bc19899b570a jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata @@ -15,32 +15,58 @@ templates: metrics: - Accuracy original_task: false - name: single_sentence_bad_yes_no + name: single_sentence_bad_no_yes reference: '' - 3b2b6cfb-5b6a-41a8-9aec-35fdca3044e9: !Template - answer_choices: 1 ||| 2 - id: 3b2b6cfb-5b6a-41a8-9aec-35fdca3044e9 + 65bef5ea-aa86-4a1f-bb22-0def0bd2055a: !Template + answer_choices: Yes ||| No + id: 65bef5ea-aa86-4a1f-bb22-0def0bd2055a + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 8882dae4-ed47-4d66-ac20-8c4c585abdf0: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 8882dae4-ed47-4d66-ac20-8c4c585abdf0 + jinja: '" ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + ae1ca2f2-4ca2-4d6e-bd32-4167e547df00: !Template + answer_choices: A ||| B + id: ae1ca2f2-4ca2-4d6e-bd32-4167e547df00 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -54,14 +80,14 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: better_B_A reference: '' - 8e9baedd-f5cd-4a8c-b66e-3f40149f38cf: !Template + b2dbdabb-af0c-42db-afe1-789e9453def5: !Template answer_choices: 1 ||| 2 - id: 8e9baedd-f5cd-4a8c-b66e-3f40149f38cf + id: b2dbdabb-af0c-42db-afe1-789e9453def5 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -93,58 +119,53 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: grammatical_between_1_2 reference: '' - 8f282b1d-077e-4dc6-9f42-0a466bf43a87: !Template + bcd4116b-ca3c-4770-b29c-aa3843492b85: !Template answer_choices: Yes ||| No - id: 8f282b1d-077e-4dc6-9f42-0a466bf43a87 + id: bcd4116b-ca3c-4770-b29c-aa3843492b85 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_bad_no_yes + name: single_sentence_good_no_yes reference: '' - a08cff29-52d7-4897-804a-39fc6743a1a2: !Template - answer_choices: Yes ||| No - id: a08cff29-52d7-4897-804a-39fc6743a1a2 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + c073ee55-fe16-4bd6-ba92-2942a73ca136: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: c073ee55-fe16-4bd6-ba92-2942a73ca136 + jinja: ' ||| {{ sentence_good }}' metadata: !TemplateMetadata - choices_in_prompt: true + choices_in_prompt: false metrics: - Accuracy - original_task: false - name: single_sentence_good_no_yes + original_task: true + name: true_null_prompt reference: '' - be3b9b9d-b82c-4450-87a8-3fbd0aa1230c: !Template - answer_choices: A ||| B - id: be3b9b9d-b82c-4450-87a8-3fbd0aa1230c + d2917209-4d2b-4468-abe7-2ad11650dcf9: !Template + answer_choices: 1 ||| 2 + id: d2917209-4d2b-4468-abe7-2ad11650dcf9 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -164,11 +185,11 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: grammatical_which_one_1_2 reference: '' - c5422144-4e75-4c66-84ca-57fcc4233a5b: !Template + e5445e04-7d31-47ff-b225-ebae6a991f2c: !Template answer_choices: A ||| B - id: c5422144-4e75-4c66-84ca-57fcc4233a5b + id: e5445e04-7d31-47ff-b225-ebae6a991f2c jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -187,7 +208,7 @@ templates: {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order + Which one is a better sentence of English, A or B? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -202,31 +223,32 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: better_A_B reference: '' - ca8ff566-c1fa-4ae1-acfc-5110cdf477eb: !Template + ed2d4629-29b0-44cb-b459-06aa4c58a66b: !Template answer_choices: A ||| B - id: ca8ff566-c1fa-4ae1-acfc-5110cdf477eb + id: ed2d4629-29b0-44cb-b459-06aa4c58a66b jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -240,14 +262,30 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: grammatical_which_one_A_B reference: '' - d4eb6764-77bf-4e3c-9024-d276dfd7880f: !Template + f3f5612c-61fe-431a-87c1-9092f94780bb: !Template + answer_choices: Yes ||| No + id: f3f5612c-61fe-431a-87c1-9092f94780bb + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + ffdd944c-ebee-46b9-9434-869197a153ca: !Template answer_choices: A ||| B - id: d4eb6764-77bf-4e3c-9024-d276dfd7880f + id: ffdd944c-ebee-46b9-9434-869197a153ca jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -279,21 +317,5 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B - reference: '' - d7545db3-84b6-4d0e-93e0-9d77905ac656: !Template - answer_choices: Yes ||| No - id: d7545db3-84b6-4d0e-93e0-9d77905ac656 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no + name: grammatical_between_A_B reference: '' diff --git a/promptsource/templates/blimp/determiner_noun_agreement_with_adj_irregular_1/templates.yaml b/promptsource/templates/blimp/determiner_noun_agreement_with_adj_irregular_1/templates.yaml index a32f5b483..cc973cb3c 100644 --- a/promptsource/templates/blimp/determiner_noun_agreement_with_adj_irregular_1/templates.yaml +++ b/promptsource/templates/blimp/determiner_noun_agreement_with_adj_irregular_1/templates.yaml @@ -1,29 +1,30 @@ dataset: blimp subset: determiner_noun_agreement_with_adj_irregular_1 templates: - 41a3a39d-5c4d-4988-a800-b03d740498bd: !Template - answer_choices: A ||| B - id: 41a3a39d-5c4d-4988-a800-b03d740498bd + 063a3b4f-5603-4976-99ef-30f335a92a4c: !Template + answer_choices: 1 ||| 2 + id: 063a3b4f-5603-4976-99ef-30f335a92a4c jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -37,65 +38,22 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: grammatical_between_1_2 reference: '' - 5f53637e-06a1-426c-992a-62f6fcb6cabc: !Template - answer_choices: Yes ||| No - id: 5f53637e-06a1-426c-992a-62f6fcb6cabc - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + 0c7a8539-45d5-4281-8650-b26ab0b84e9f: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 0c7a8539-45d5-4281-8650-b26ab0b84e9f + jinja: ' ||| {{ sentence_good }}' metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes - reference: '' - 73662fd0-ec0e-40b3-bc9e-fa35fba20815: !Template - answer_choices: A ||| B - id: 73662fd0-ec0e-40b3-bc9e-fa35fba20815 - jinja: '{% set shuffled_order = [0, 1] | random %} - - Read the following two sentences. - - {% if shuffled_order == 0 %} - - Sentence A: {{ sentence_good }} - - Sentence B: {{ sentence_bad }} - - {% else %} - - Sentence A: {{ sentence_bad }} - - Sentence B: {{ sentence_good }} - - {% endif %} - - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} - - {% endif %}' - metadata: !TemplateMetadata - choices_in_prompt: true + choices_in_prompt: false metrics: - Accuracy original_task: true - name: better_B_A + name: true_null_prompt reference: '' - 7f0fda4d-a15d-4dc2-a516-31aa0e099ade: !Template + 11b62273-ecf0-4677-8a33-21800a7bdecb: !Template answer_choices: Yes ||| No - id: 7f0fda4d-a15d-4dc2-a516-31aa0e099ade + id: 11b62273-ecf0-4677-8a33-21800a7bdecb jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -109,9 +67,9 @@ templates: original_task: false name: single_sentence_bad_no_yes reference: '' - 981b7c2e-139e-448e-b0e7-d55b75497e37: !Template + 1dfe5621-892e-4708-9a07-575d48b7cfb3: !Template answer_choices: Yes ||| No - id: 981b7c2e-139e-448e-b0e7-d55b75497e37 + id: 1dfe5621-892e-4708-9a07-575d48b7cfb3 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -125,9 +83,9 @@ templates: original_task: false name: single_sentence_good_yes_no reference: '' - a72f3304-f283-4449-8298-576e8446d1fd: !Template + 230f386f-75fe-47c6-9790-dad308ebb798: !Template answer_choices: 1 ||| 2 - id: a72f3304-f283-4449-8298-576e8446d1fd + id: 230f386f-75fe-47c6-9790-dad308ebb798 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer 1 or 2. @@ -164,30 +122,29 @@ templates: original_task: true name: grammatical_which_one_1_2 reference: '' - cebaa4c0-dadb-4e94-8f6f-896ea8a1f1dc: !Template - answer_choices: 1 ||| 2 - id: cebaa4c0-dadb-4e94-8f6f-896ea8a1f1dc + 2cbf40f2-612d-4f91-a3b3-52867c1e3be8: !Template + answer_choices: A ||| B + id: 2cbf40f2-612d-4f91-a3b3-52867c1e3be8 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -201,14 +158,14 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: better_A_B reference: '' - d2b12553-0384-4aee-b357-0fd004d7df11: !Template + 42417d55-e14b-4fe6-ad46-133a1d93d43f: !Template answer_choices: A ||| B - id: d2b12553-0384-4aee-b357-0fd004d7df11 + id: 42417d55-e14b-4fe6-ad46-133a1d93d43f jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -240,14 +197,52 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: grammatical_which_one_A_B reference: '' - f127b1d1-8bc9-46b3-ad43-ef83a6182721: !Template + 56380f29-db99-416d-92d5-044298d030bf: !Template answer_choices: A ||| B - id: f127b1d1-8bc9-46b3-ad43-ef83a6182721 + id: 56380f29-db99-416d-92d5-044298d030bf jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: better_B_A + reference: '' + a0d63188-a913-4ad9-bbc8-3b315d23dee1: !Template + answer_choices: A ||| B + id: a0d63188-a913-4ad9-bbc8-3b315d23dee1 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -279,11 +274,11 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: grammatical_between_A_B reference: '' - f17dccfc-874d-4073-b0da-934539891bca: !Template + acd86c1a-5fb1-4581-b455-231d325de841: !Template answer_choices: Yes ||| No - id: f17dccfc-874d-4073-b0da-934539891bca + id: acd86c1a-5fb1-4581-b455-231d325de841 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -297,3 +292,30 @@ templates: original_task: false name: single_sentence_bad_yes_no reference: '' + c70028f7-f29a-4d7f-bd0a-eb584855d7b4: !Template + answer_choices: Yes ||| No + id: c70028f7-f29a-4d7f-bd0a-eb584855d7b4 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes + reference: '' + ed1d67ab-0a04-4634-ac00-faa5eaabde4b: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: ed1d67ab-0a04-4634-ac00-faa5eaabde4b + jinja: '" ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' diff --git a/promptsource/templates/blimp/determiner_noun_agreement_with_adj_irregular_2/templates.yaml b/promptsource/templates/blimp/determiner_noun_agreement_with_adj_irregular_2/templates.yaml index 1e69784b5..215cc41bd 100644 --- a/promptsource/templates/blimp/determiner_noun_agreement_with_adj_irregular_2/templates.yaml +++ b/promptsource/templates/blimp/determiner_noun_agreement_with_adj_irregular_2/templates.yaml @@ -1,40 +1,40 @@ dataset: blimp subset: determiner_noun_agreement_with_adj_irregular_2 templates: - 2354b1e1-f868-4332-bce7-66041e126ae8: !Template + 0cb0767a-40f3-4ff0-aebe-e1a5c33c976d: !Template answer_choices: Yes ||| No - id: 2354b1e1-f868-4332-bce7-66041e126ae8 + id: 0cb0767a-40f3-4ff0-aebe-e1a5c33c976d jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_good_yes_no + name: single_sentence_bad_no_yes reference: '' - 581fa102-823c-4a0e-a510-f51fbf2bdece: !Template - answer_choices: A ||| B - id: 581fa102-823c-4a0e-a510-f51fbf2bdece + 20257e24-2ef9-419b-a74d-b96cef4fd3e1: !Template + answer_choices: 1 ||| 2 + id: 20257e24-2ef9-419b-a74d-b96cef4fd3e1 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -54,26 +54,64 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: grammatical_which_one_1_2 reference: '' - 5a66a33a-fd03-4b54-b367-24b7bb0ec423: !Template + 35ef4d06-1b5d-4191-8599-df70048bc208: !Template answer_choices: A ||| B - id: 5a66a33a-fd03-4b54-b367-24b7bb0ec423 + id: 35ef4d06-1b5d-4191-8599-df70048bc208 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: better_A_B + reference: '' + 4297cd21-25af-4c96-98ab-760591b3bff4: !Template + answer_choices: 1 ||| 2 + id: 4297cd21-25af-4c96-98ab-760591b3bff4 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} {% endif %} @@ -93,31 +131,32 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: grammatical_between_1_2 reference: '' - 6b583fbb-5034-4ce1-81c6-58d655661e22: !Template + 5b520f6e-b2da-47bd-a01c-1d6b0d7b8d84: !Template answer_choices: A ||| B - id: 6b583fbb-5034-4ce1-81c6-58d655661e22 + id: 5b520f6e-b2da-47bd-a01c-1d6b0d7b8d84 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -131,11 +170,11 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: grammatical_which_one_A_B reference: '' - 75afa7ac-1293-4ac0-b8fb-a6ed466e4329: !Template + 7baeee4a-29ee-4c12-b205-a7348c8ea042: !Template answer_choices: Yes ||| No - id: 75afa7ac-1293-4ac0-b8fb-a6ed466e4329 + id: 7baeee4a-29ee-4c12-b205-a7348c8ea042 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -149,13 +188,24 @@ templates: original_task: false name: single_sentence_good_no_yes reference: '' - 823a5fa9-8c51-4d59-958f-eedf890797a6: !Template + 7de33be7-58d4-4760-b905-1f48db075936: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 7de33be7-58d4-4760-b905-1f48db075936 + jinja: '" ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + 8f07a059-1366-44a0-b728-620535232ac3: !Template answer_choices: Yes ||| No - id: 823a5fa9-8c51-4d59-958f-eedf890797a6 + id: 8f07a059-1366-44a0-b728-620535232ac3 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata @@ -163,26 +213,26 @@ templates: metrics: - Accuracy original_task: false - name: single_sentence_bad_no_yes + name: single_sentence_bad_yes_no reference: '' - 97e75645-7f2b-4490-8959-06de683d02a6: !Template - answer_choices: 1 ||| 2 - id: 97e75645-7f2b-4490-8959-06de683d02a6 + 96eb1910-db1c-4917-9f17-3b7be652aa27: !Template + answer_choices: A ||| B + id: 96eb1910-db1c-4917-9f17-3b7be652aa27 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -202,27 +252,22 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: grammatical_between_A_B reference: '' - ced385f4-6a92-4f90-813f-7cc2cc47ee4b: !Template - answer_choices: Yes ||| No - id: ced385f4-6a92-4f90-813f-7cc2cc47ee4b - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' + b8885a4a-7fac-4208-8f75-02a92e86c7db: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: b8885a4a-7fac-4208-8f75-02a92e86c7db + jinja: ' ||| {{ sentence_good }}' metadata: !TemplateMetadata - choices_in_prompt: true + choices_in_prompt: false metrics: - Accuracy - original_task: false - name: single_sentence_bad_yes_no + original_task: true + name: true_null_prompt reference: '' - e8f7caae-a9b3-434e-a033-0405636d26d7: !Template + ee080fdb-8b82-4226-98c2-793a39b13d64: !Template answer_choices: A ||| B - id: e8f7caae-a9b3-434e-a033-0405636d26d7 + id: ee080fdb-8b82-4226-98c2-793a39b13d64 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -258,42 +303,19 @@ templates: original_task: true name: better_B_A reference: '' - fd8b6ded-6cac-452d-89fc-7e18fb70658d: !Template - answer_choices: 1 ||| 2 - id: fd8b6ded-6cac-452d-89fc-7e18fb70658d - jinja: '{% set shuffled_order = [0, 1] | random %} - - Between the two sentences, which one is grammatical? Please answer 1 or 2. - - {% if shuffled_order == 0 %} - - 1: {{ sentence_good }} - - 2: {{ sentence_bad }} - - {% else %} - - 1: {{ sentence_bad }} - - 2: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} + f3a2b574-0c67-4d66-9772-54b13ecafc9b: !Template + answer_choices: Yes ||| No + id: f3a2b574-0c67-4d66-9772-54b13ecafc9b + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - {% endif %}' + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: grammatical_between_1_2 + original_task: false + name: single_sentence_good_yes_no reference: '' diff --git a/promptsource/templates/blimp/determiner_noun_agreement_with_adjective_1/templates.yaml b/promptsource/templates/blimp/determiner_noun_agreement_with_adjective_1/templates.yaml index ba83fe257..438514cc4 100644 --- a/promptsource/templates/blimp/determiner_noun_agreement_with_adjective_1/templates.yaml +++ b/promptsource/templates/blimp/determiner_noun_agreement_with_adjective_1/templates.yaml @@ -1,9 +1,9 @@ dataset: blimp subset: determiner_noun_agreement_with_adjective_1 templates: - 177fdeaa-4de1-4e8f-a325-23ad33e86376: !Template + 452a4bff-9c60-49b3-85b4-8ae6267fb36b: !Template answer_choices: Yes ||| No - id: 177fdeaa-4de1-4e8f-a325-23ad33e86376 + id: 452a4bff-9c60-49b3-85b4-8ae6267fb36b jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -17,51 +17,23 @@ templates: original_task: false name: single_sentence_bad_yes_no reference: '' - 33bfb629-a0c2-469e-b54d-571f9f874f46: !Template - answer_choices: A ||| B - id: 33bfb629-a0c2-469e-b54d-571f9f874f46 - jinja: '{% set shuffled_order = [0, 1] | random %} - - Between the two sentences, which one is grammatical? Please answer A or B. - - {% if shuffled_order == 0 %} - - A: {{ sentence_good }} - - B: {{ sentence_bad }} - - {% else %} - - A: {{ sentence_bad }} - - B: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} - - {% endif %}' + 5072949c-2ceb-48e5-be2c-d0175ac71207: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 5072949c-2ceb-48e5-be2c-d0175ac71207 + jinja: '" ||| {{ sentence_good }}' metadata: !TemplateMetadata - choices_in_prompt: true + choices_in_prompt: false metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: null_prompt reference: '' - 9b67bc74-7b46-4987-bf8f-9c0478eddd9c: !Template + 517dda30-4692-43f0-b448-ba30836314e0: !Template answer_choices: A ||| B - id: 9b67bc74-7b46-4987-bf8f-9c0478eddd9c + id: 517dda30-4692-43f0-b448-ba30836314e0 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -93,11 +65,11 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: grammatical_between_A_B reference: '' - a458d8d5-ca2f-4cf9-8ecf-00151c3c072e: !Template + 593639c9-5d3e-4cce-9758-be2003dbf8cc: !Template answer_choices: Yes ||| No - id: a458d8d5-ca2f-4cf9-8ecf-00151c3c072e + id: 593639c9-5d3e-4cce-9758-be2003dbf8cc jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -111,9 +83,9 @@ templates: original_task: false name: single_sentence_good_yes_no reference: '' - a7e580a4-b5e3-42e2-879c-7a20bd58c64d: !Template + 5b43a31c-1173-485a-820b-71087d79b0bc: !Template answer_choices: A ||| B - id: a7e580a4-b5e3-42e2-879c-7a20bd58c64d + id: 5b43a31c-1173-485a-820b-71087d79b0bc jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -149,24 +121,24 @@ templates: original_task: true name: better_A_B reference: '' - b1717065-b7f9-471d-ab1b-630c89bb0525: !Template - answer_choices: 1 ||| 2 - id: b1717065-b7f9-471d-ab1b-630c89bb0525 + 6a4dd847-92d0-4bdc-88ac-62d8b3a8aa31: !Template + answer_choices: A ||| B + id: 6a4dd847-92d0-4bdc-88ac-62d8b3a8aa31 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -186,14 +158,41 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: grammatical_which_one_A_B + reference: '' + 79fb5615-290d-4435-8d1a-bda7afd69e9b: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 79fb5615-290d-4435-8d1a-bda7afd69e9b + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: true_null_prompt + reference: '' + b03e0373-24f7-4cf3-9de0-ce770c7c3e2f: !Template + answer_choices: Yes ||| No + id: b03e0373-24f7-4cf3-9de0-ce770c7c3e2f + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes reference: '' - c9aff3a4-145d-4e6a-91b0-bb482dd40856: !Template + c5191b94-65b9-4562-9438-84bfb78bd4c8: !Template answer_choices: 1 ||| 2 - id: c9aff3a4-145d-4e6a-91b0-bb482dd40856 + id: c5191b94-65b9-4562-9438-84bfb78bd4c8 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -225,43 +224,11 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 - reference: '' - d0273898-368a-497f-bfde-e0625f3603fd: !Template - answer_choices: Yes ||| No - id: d0273898-368a-497f-bfde-e0625f3603fd - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes - reference: '' - f14441f0-315a-45e0-960d-b4ecbb6d661a: !Template - answer_choices: Yes ||| No - id: f14441f0-315a-45e0-960d-b4ecbb6d661a - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes + name: grammatical_between_1_2 reference: '' - fb0d4859-3ca3-40cd-8328-8eb492283cfb: !Template + e0d4bab2-7fd4-4b4b-a378-beaafd9f9e49: !Template answer_choices: A ||| B - id: fb0d4859-3ca3-40cd-8328-8eb492283cfb + id: e0d4bab2-7fd4-4b4b-a378-beaafd9f9e49 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -297,3 +264,58 @@ templates: original_task: true name: better_B_A reference: '' + e1ec22cd-feb1-404f-8124-98e0874533a4: !Template + answer_choices: 1 ||| 2 + id: e1ec22cd-feb1-404f-8124-98e0874533a4 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_1_2 + reference: '' + f17e6002-bd89-4c21-b631-0cb20faa5912: !Template + answer_choices: Yes ||| No + id: f17e6002-bd89-4c21-b631-0cb20faa5912 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes + reference: '' diff --git a/promptsource/templates/blimp/distractor_agreement_relational_noun/templates.yaml b/promptsource/templates/blimp/distractor_agreement_relational_noun/templates.yaml index df724fda3..8309459d6 100644 --- a/promptsource/templates/blimp/distractor_agreement_relational_noun/templates.yaml +++ b/promptsource/templates/blimp/distractor_agreement_relational_noun/templates.yaml @@ -1,62 +1,68 @@ dataset: blimp subset: distractor_agreement_relational_noun templates: - 094bd674-442c-4613-8527-f4b908166fca: !Template - answer_choices: Yes ||| No - id: 094bd674-442c-4613-8527-f4b908166fca - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + 028695a2-4f72-4aae-af65-0f6373512a76: !Template + answer_choices: 1 ||| 2 + id: 028695a2-4f72-4aae-af65-0f6373512a76 + jinja: '{% set shuffled_order = [0, 1] | random %} - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no - reference: '' - 113bf501-a714-4a8d-a653-2f4b2f2f6ec3: !Template - answer_choices: Yes ||| No - id: 113bf501-a714-4a8d-a653-2f4b2f2f6ec3 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + Which one of the following sentences is grammatical? Please answer 1 or 2. - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: false - name: single_sentence_good_no_yes + original_task: true + name: grammatical_which_one_1_2 reference: '' - 1d8c3ab0-adc6-4f6d-8b6b-b612dae56f9d: !Template + 2024f0d9-491c-46fd-b8f5-53e34db8853d: !Template answer_choices: A ||| B - id: 1d8c3ab0-adc6-4f6d-8b6b-b612dae56f9d + id: 2024f0d9-491c-46fd-b8f5-53e34db8853d jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -70,14 +76,14 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: better_A_B reference: '' - 4eef0ebe-6760-4a57-9492-2076915cadae: !Template + 2c62cdb9-adc7-458c-9b28-1120c1f98b7f: !Template answer_choices: A ||| B - id: 4eef0ebe-6760-4a57-9492-2076915cadae + id: 2c62cdb9-adc7-458c-9b28-1120c1f98b7f jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -109,47 +115,32 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B - reference: '' - 7fc778b4-aebe-40f3-b270-620630d7cde5: !Template - answer_choices: Yes ||| No - id: 7fc778b4-aebe-40f3-b270-620630d7cde5 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes + name: grammatical_between_A_B reference: '' - a282f89a-c0d8-421b-bdbb-4fa0bb7d6a91: !Template + 3b5ade02-1f86-48a9-b194-71c3c5033fcd: !Template answer_choices: A ||| B - id: a282f89a-c0d8-421b-bdbb-4fa0bb7d6a91 + id: 3b5ade02-1f86-48a9-b194-71c3c5033fcd jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -163,11 +154,27 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: grammatical_which_one_A_B + reference: '' + 61fb2ae7-ea43-4f13-9e72-9f05e98668d6: !Template + answer_choices: Yes ||| No + id: 61fb2ae7-ea43-4f13-9e72-9f05e98668d6 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no reference: '' - bc3cb2f3-1ca6-42a8-81d0-8e36dd1cc13b: !Template + 7572752b-cf67-4e8d-94d9-eb85b87c7ad1: !Template answer_choices: 1 ||| 2 - id: bc3cb2f3-1ca6-42a8-81d0-8e36dd1cc13b + id: 7572752b-cf67-4e8d-94d9-eb85b87c7ad1 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -204,13 +211,13 @@ templates: original_task: true name: grammatical_between_1_2 reference: '' - c774c7b0-7868-45f9-a48c-3fae42eed917: !Template + 795af14f-7b5b-402a-bb76-c5db559618a7: !Template answer_choices: Yes ||| No - id: c774c7b0-7868-45f9-a48c-3fae42eed917 + id: 795af14f-7b5b-402a-bb76-c5db559618a7 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -218,11 +225,38 @@ templates: metrics: - Accuracy original_task: false - name: single_sentence_good_yes_no + name: single_sentence_good_no_yes reference: '' - c986eb7e-8e8f-46f3-b421-c18199d1ff1f: !Template + 8f7a259e-840f-441f-9746-6061c5b7658b: !Template + answer_choices: Yes ||| No + id: 8f7a259e-840f-441f-9746-6061c5b7658b + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes + reference: '' + 97afeded-6b3d-4e53-91ba-b4dccf126e83: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 97afeded-6b3d-4e53-91ba-b4dccf126e83 + jinja: '" ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + a0b01db7-8fe8-4720-939b-bf71f35d370c: !Template answer_choices: A ||| B - id: c986eb7e-8e8f-46f3-b421-c18199d1ff1f + id: a0b01db7-8fe8-4720-939b-bf71f35d370c jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -258,42 +292,30 @@ templates: original_task: true name: better_B_A reference: '' - ddd82457-5ab4-41ec-a44e-d00be9bf3727: !Template - answer_choices: 1 ||| 2 - id: ddd82457-5ab4-41ec-a44e-d00be9bf3727 - jinja: '{% set shuffled_order = [0, 1] | random %} - - Which one of the following sentences is grammatical? Please answer 1 or 2. - - {% if shuffled_order == 0 %} - - 1: {{ sentence_good }} - - 2: {{ sentence_bad }} - - {% else %} - - 1: {{ sentence_bad }} - - 2: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} + a19d24a0-c9d0-46ba-a1dc-eef505ab211a: !Template + answer_choices: Yes ||| No + id: a19d24a0-c9d0-46ba-a1dc-eef505ab211a + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - {% endif %}' + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + f54a1fe1-20a6-44fc-b3de-245d8500e36f: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: f54a1fe1-20a6-44fc-b3de-245d8500e36f + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: true_null_prompt reference: '' diff --git a/promptsource/templates/blimp/distractor_agreement_relative_clause/templates.yaml b/promptsource/templates/blimp/distractor_agreement_relative_clause/templates.yaml index 8046b7c8f..9591fd05d 100644 --- a/promptsource/templates/blimp/distractor_agreement_relative_clause/templates.yaml +++ b/promptsource/templates/blimp/distractor_agreement_relative_clause/templates.yaml @@ -1,61 +1,57 @@ dataset: blimp subset: distractor_agreement_relative_clause templates: - 14c4b091-49b7-4c98-b37a-71af6ac4811c: !Template - answer_choices: Yes ||| No - id: 14c4b091-49b7-4c98-b37a-71af6ac4811c - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + 02aa781c-0c1a-4200-91a4-3f6976461a27: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 02aa781c-0c1a-4200-91a4-3f6976461a27 + jinja: '" ||| {{ sentence_good }}' metadata: !TemplateMetadata - choices_in_prompt: true + choices_in_prompt: false metrics: - Accuracy - original_task: false - name: single_sentence_good_no_yes + original_task: true + name: null_prompt reference: '' - 2f57ebd3-4617-46f4-96d7-afdacbd72469: !Template + 0db6fa0a-e41c-4c0e-9fd6-03e86cb7f546: !Template answer_choices: Yes ||| No - id: 2f57ebd3-4617-46f4-96d7-afdacbd72469 + id: 0db6fa0a-e41c-4c0e-9fd6-03e86cb7f546 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_bad_no_yes + name: single_sentence_good_no_yes reference: '' - 4ad745ea-017b-41c5-a918-4c56d1daf41d: !Template - answer_choices: A ||| B - id: 4ad745ea-017b-41c5-a918-4c56d1daf41d + 126a54c7-4ccd-4114-be6d-b21fca2f6bb6: !Template + answer_choices: 1 ||| 2 + id: 126a54c7-4ccd-4114-be6d-b21fca2f6bb6 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -69,14 +65,14 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: grammatical_between_1_2 reference: '' - 4d1c247f-4d4c-4e1e-88cf-445eea5ea1e3: !Template + 16b1cd6b-048a-40a5-af93-32b5f4eb2820: !Template answer_choices: A ||| B - id: 4d1c247f-4d4c-4e1e-88cf-445eea5ea1e3 + id: 16b1cd6b-048a-40a5-af93-32b5f4eb2820 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -108,31 +104,75 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: grammatical_between_A_B + reference: '' + 59f00aca-ba2e-4e7e-a73e-9571cc819313: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 59f00aca-ba2e-4e7e-a73e-9571cc819313 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: true_null_prompt + reference: '' + 620c3f0b-ebec-45c0-bcd2-9c8d43471f41: !Template + answer_choices: Yes ||| No + id: 620c3f0b-ebec-45c0-bcd2-9c8d43471f41 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes reference: '' - 55bfe218-6eab-40ee-a596-ec4b52956c83: !Template + 9315385b-5010-468a-9c4d-dd343bc0148b: !Template + answer_choices: Yes ||| No + id: 9315385b-5010-468a-9c4d-dd343bc0148b + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + a5d590e5-389e-4244-a460-fc301d2b4866: !Template answer_choices: A ||| B - id: 55bfe218-6eab-40ee-a596-ec4b52956c83 + id: a5d590e5-389e-4244-a460-fc301d2b4866 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -146,48 +186,31 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A - reference: '' - 5768b427-5f4d-4b29-b89f-f4088fb47731: !Template - answer_choices: Yes ||| No - id: 5768b427-5f4d-4b29-b89f-f4088fb47731 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no + name: grammatical_which_one_A_B reference: '' - 69563a61-ab33-4d02-92e2-04d848bdfdef: !Template - answer_choices: 1 ||| 2 - id: 69563a61-ab33-4d02-92e2-04d848bdfdef + b09975e3-7e3e-42e5-a623-985ce049eda2: !Template + answer_choices: A ||| B + id: b09975e3-7e3e-42e5-a623-985ce049eda2 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -201,11 +224,11 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: better_A_B reference: '' - bffb193d-70a0-4790-abc2-40c98314019b: !Template + d3a10065-5921-4bdf-944a-b0d062911df0: !Template answer_choices: Yes ||| No - id: bffb193d-70a0-4790-abc2-40c98314019b + id: d3a10065-5921-4bdf-944a-b0d062911df0 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -219,24 +242,24 @@ templates: original_task: false name: single_sentence_bad_yes_no reference: '' - cde5ffde-c1fb-4b8f-b52e-09ae2269a509: !Template - answer_choices: A ||| B - id: cde5ffde-c1fb-4b8f-b52e-09ae2269a509 + e62391a2-d78b-4d00-b821-0340e39b75b1: !Template + answer_choices: 1 ||| 2 + id: e62391a2-d78b-4d00-b821-0340e39b75b1 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -256,32 +279,31 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: grammatical_which_one_1_2 reference: '' - fa717a13-e152-4131-925a-5a324aed53df: !Template - answer_choices: 1 ||| 2 - id: fa717a13-e152-4131-925a-5a324aed53df + fea7b9d3-46af-423f-aad8-8db1653fb086: !Template + answer_choices: A ||| B + id: fea7b9d3-46af-423f-aad8-8db1653fb086 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -295,5 +317,5 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: better_B_A reference: '' diff --git a/promptsource/templates/blimp/drop_argument/templates.yaml b/promptsource/templates/blimp/drop_argument/templates.yaml index bca1a0de4..50c3b2517 100644 --- a/promptsource/templates/blimp/drop_argument/templates.yaml +++ b/promptsource/templates/blimp/drop_argument/templates.yaml @@ -1,9 +1,36 @@ dataset: blimp subset: drop_argument templates: - 137031cb-2dcb-4881-86ea-7d221b84607c: !Template + 0141184b-09b2-4d5a-9f76-9d4db8612e65: !Template answer_choices: Yes ||| No - id: 137031cb-2dcb-4881-86ea-7d221b84607c + id: 0141184b-09b2-4d5a-9f76-9d4db8612e65 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes + reference: '' + 1441372d-f77d-46b4-b185-89b8855282ca: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 1441372d-f77d-46b4-b185-89b8855282ca + jinja: '" ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + 3524a6ac-4e76-4252-b0fd-91f6be66366d: !Template + answer_choices: Yes ||| No + id: 3524a6ac-4e76-4252-b0fd-91f6be66366d jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -17,12 +44,28 @@ templates: original_task: false name: single_sentence_bad_yes_no reference: '' - 1a04249a-4538-4830-8e7c-a516727c159d: !Template + 3885563f-e59b-4e3e-b086-3312c4ce2544: !Template + answer_choices: Yes ||| No + id: 3885563f-e59b-4e3e-b086-3312c4ce2544 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes + reference: '' + 394a377f-2e86-4aa5-8e13-10223b9e1a18: !Template answer_choices: 1 ||| 2 - id: 1a04249a-4538-4830-8e7c-a516727c159d + id: 394a377f-2e86-4aa5-8e13-10223b9e1a18 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -54,32 +97,47 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: grammatical_which_one_1_2 reference: '' - 263e3de0-32b1-4640-a6e1-2bca0deabdb1: !Template + 6e1f6c6e-1bc2-46fb-9cf4-bdca420c20ac: !Template + answer_choices: Yes ||| No + id: 6e1f6c6e-1bc2-46fb-9cf4-bdca420c20ac + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 9f63829b-12e5-4ad5-95f0-0ac57d005567: !Template answer_choices: A ||| B - id: 263e3de0-32b1-4640-a6e1-2bca0deabdb1 + id: 9f63829b-12e5-4ad5-95f0-0ac57d005567 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -93,14 +151,25 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: better_A_B reference: '' - 34f51242-e535-4213-9f8f-7b54c54f64f1: !Template + 9fabf622-f791-408e-be16-fa47fae7134e: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 9fabf622-f791-408e-be16-fa47fae7134e + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: true_null_prompt + reference: '' + a769e479-75dc-4073-9d17-ab4684040ee7: !Template answer_choices: A ||| B - id: 34f51242-e535-4213-9f8f-7b54c54f64f1 + id: a769e479-75dc-4073-9d17-ab4684040ee7 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -132,31 +201,32 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: grammatical_which_one_A_B reference: '' - 3f6a0b47-5606-43f7-a08a-c6eb57d1c6d1: !Template + d4e1cb9c-1402-4583-8103-6708b39354c8: !Template answer_choices: A ||| B - id: 3f6a0b47-5606-43f7-a08a-c6eb57d1c6d1 + id: d4e1cb9c-1402-4583-8103-6708b39354c8 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -170,64 +240,31 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B - reference: '' - 957ef5ba-fe78-48f3-9388-5e6f087b1eaf: !Template - answer_choices: Yes ||| No - id: 957ef5ba-fe78-48f3-9388-5e6f087b1eaf - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes - reference: '' - 9c7271c8-ed8d-4850-b0fb-62116d7c90a7: !Template - answer_choices: Yes ||| No - id: 9c7271c8-ed8d-4850-b0fb-62116d7c90a7 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes + name: grammatical_between_A_B reference: '' - def99203-248c-4ff5-88c7-f40a6dd59c3d: !Template - answer_choices: 1 ||| 2 - id: def99203-248c-4ff5-88c7-f40a6dd59c3d + e08a3cbf-3938-4409-9a76-e8d46590c5e2: !Template + answer_choices: A ||| B + id: e08a3cbf-3938-4409-9a76-e8d46590c5e2 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -241,31 +278,32 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: better_B_A reference: '' - e2e39a00-9921-4068-a5d3-1d33e7afcefe: !Template - answer_choices: A ||| B - id: e2e39a00-9921-4068-a5d3-1d33e7afcefe + e701cdc9-f85b-4bee-99f9-caaf18de060c: !Template + answer_choices: 1 ||| 2 + id: e701cdc9-f85b-4bee-99f9-caaf18de060c jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -279,21 +317,5 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A - reference: '' - ea2c6e6b-35c7-4cb1-b3dd-55638d32c856: !Template - answer_choices: Yes ||| No - id: ea2c6e6b-35c7-4cb1-b3dd-55638d32c856 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no + name: grammatical_between_1_2 reference: '' diff --git a/promptsource/templates/blimp/ellipsis_n_bar_1/templates.yaml b/promptsource/templates/blimp/ellipsis_n_bar_1/templates.yaml index ac2ae42c2..978816f61 100644 --- a/promptsource/templates/blimp/ellipsis_n_bar_1/templates.yaml +++ b/promptsource/templates/blimp/ellipsis_n_bar_1/templates.yaml @@ -1,9 +1,41 @@ dataset: blimp subset: ellipsis_n_bar_1 templates: - 19c0226d-73dd-44fc-9884-23fa7b705705: !Template + 07e32c6c-e950-4e8a-b401-33b11a70f31c: !Template + answer_choices: Yes ||| No + id: 07e32c6c-e950-4e8a-b401-33b11a70f31c + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes + reference: '' + 3f44acfa-cb27-4366-aad6-610556aee656: !Template + answer_choices: Yes ||| No + id: 3f44acfa-cb27-4366-aad6-610556aee656 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 59745b65-3cc9-4634-832e-448f9dc1f1ac: !Template answer_choices: A ||| B - id: 19c0226d-73dd-44fc-9884-23fa7b705705 + id: 59745b65-3cc9-4634-832e-448f9dc1f1ac jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -39,40 +71,24 @@ templates: original_task: true name: better_B_A reference: '' - 3deb1c9e-7141-4aea-b88c-3573b88df02b: !Template - answer_choices: Yes ||| No - id: 3deb1c9e-7141-4aea-b88c-3573b88df02b - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no - reference: '' - 50c8864d-8d10-41dc-b77d-355f80e5c45b: !Template - answer_choices: A ||| B - id: 50c8864d-8d10-41dc-b77d-355f80e5c45b + 7e88d1d7-fdd7-4266-8779-4027fbfbd85c: !Template + answer_choices: 1 ||| 2 + id: 7e88d1d7-fdd7-4266-8779-4027fbfbd85c jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -92,27 +108,22 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: grammatical_between_1_2 reference: '' - a060f971-64cd-40f7-98dc-f7d369877941: !Template - answer_choices: Yes ||| No - id: a060f971-64cd-40f7-98dc-f7d369877941 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + 8d6391b8-b14c-4568-b69d-9338315a473e: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 8d6391b8-b14c-4568-b69d-9338315a473e + jinja: '" ||| {{ sentence_good }}' metadata: !TemplateMetadata - choices_in_prompt: true + choices_in_prompt: false metrics: - Accuracy - original_task: false - name: single_sentence_good_no_yes + original_task: true + name: null_prompt reference: '' - bb0a0195-7b97-4814-b301-2649bf529be6: !Template + b0fc4064-25be-444b-81b5-ab99165f5439: !Template answer_choices: Yes ||| No - id: bb0a0195-7b97-4814-b301-2649bf529be6 + id: b0fc4064-25be-444b-81b5-ab99165f5439 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -126,9 +137,9 @@ templates: original_task: false name: single_sentence_bad_no_yes reference: '' - c9b18356-a2e6-445f-a595-c056cbda3247: !Template + b18e6caa-ed61-4af3-91ce-225e7f4a1e10: !Template answer_choices: A ||| B - id: c9b18356-a2e6-445f-a595-c056cbda3247 + id: b18e6caa-ed61-4af3-91ce-225e7f4a1e10 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -164,12 +175,12 @@ templates: original_task: true name: better_A_B reference: '' - caaafbb1-f55a-4010-ad6d-6a6c34d8ac97: !Template + b99af5e1-1fe2-4704-9c41-cb77ba55a638: !Template answer_choices: A ||| B - id: caaafbb1-f55a-4010-ad6d-6a6c34d8ac97 + id: b99af5e1-1fe2-4704-9c41-cb77ba55a638 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -201,14 +212,25 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: grammatical_between_A_B + reference: '' + ba51725d-70ee-4bd0-8337-c31cc6b43408: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: ba51725d-70ee-4bd0-8337-c31cc6b43408 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: true_null_prompt reference: '' - cb59eede-b5c3-450f-ac87-ce35bcd6f4f5: !Template + c8e1309f-4549-4412-8ffd-b2a42b4fd0ac: !Template answer_choices: 1 ||| 2 - id: cb59eede-b5c3-450f-ac87-ce35bcd6f4f5 + id: c8e1309f-4549-4412-8ffd-b2a42b4fd0ac jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -240,42 +262,26 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 - reference: '' - d3196a24-e005-422b-a9a4-d5d6327751fa: !Template - answer_choices: Yes ||| No - id: d3196a24-e005-422b-a9a4-d5d6327751fa - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no + name: grammatical_which_one_1_2 reference: '' - fcd5cc9e-fd01-46d2-97b0-8147d23e3514: !Template - answer_choices: 1 ||| 2 - id: fcd5cc9e-fd01-46d2-97b0-8147d23e3514 + ceaf2e49-0733-4905-908b-04278cd625da: !Template + answer_choices: A ||| B + id: ceaf2e49-0733-4905-908b-04278cd625da jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -295,5 +301,21 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: grammatical_which_one_A_B + reference: '' + f4e414fc-15c3-4f69-9d23-9ba74bb3bef3: !Template + answer_choices: Yes ||| No + id: f4e414fc-15c3-4f69-9d23-9ba74bb3bef3 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no reference: '' diff --git a/promptsource/templates/blimp/ellipsis_n_bar_2/templates.yaml b/promptsource/templates/blimp/ellipsis_n_bar_2/templates.yaml index 83b3175f6..5b6bdcc3d 100644 --- a/promptsource/templates/blimp/ellipsis_n_bar_2/templates.yaml +++ b/promptsource/templates/blimp/ellipsis_n_bar_2/templates.yaml @@ -1,25 +1,9 @@ dataset: blimp subset: ellipsis_n_bar_2 templates: - 3b95807f-c6e0-483e-a32b-d564c754df56: !Template - answer_choices: Yes ||| No - id: 3b95807f-c6e0-483e-a32b-d564c754df56 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes - reference: '' - 4537d356-e7e6-46c7-acff-a1b26ca23af9: !Template + 00b415b3-ad67-447c-bbc1-dcce2eba458e: !Template answer_choices: A ||| B - id: 4537d356-e7e6-46c7-acff-a1b26ca23af9 + id: 00b415b3-ad67-447c-bbc1-dcce2eba458e jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -55,62 +39,51 @@ templates: original_task: true name: better_B_A reference: '' - 4eb6609d-8070-4840-a423-d486b2f524ce: !Template - answer_choices: Yes ||| No - id: 4eb6609d-8070-4840-a423-d486b2f524ce - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' + 013058f2-938f-4370-8894-3e24e9a0937e: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 013058f2-938f-4370-8894-3e24e9a0937e + jinja: ' ||| {{ sentence_good }}' metadata: !TemplateMetadata - choices_in_prompt: true + choices_in_prompt: false metrics: - Accuracy - original_task: false - name: single_sentence_bad_no_yes + original_task: true + name: true_null_prompt reference: '' - 5db7087e-8f5a-46b6-bc65-975e677e9220: !Template - answer_choices: Yes ||| No - id: 5db7087e-8f5a-46b6-bc65-975e677e9220 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + 07f50289-db00-4bd0-a171-ebaa64fd5735: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 07f50289-db00-4bd0-a171-ebaa64fd5735 + jinja: '" ||| {{ sentence_good }}' metadata: !TemplateMetadata - choices_in_prompt: true + choices_in_prompt: false metrics: - Accuracy - original_task: false - name: single_sentence_good_yes_no + original_task: true + name: null_prompt reference: '' - 5f6ee442-9961-4f8c-ba36-32a63ec86239: !Template - answer_choices: 1 ||| 2 - id: 5f6ee442-9961-4f8c-ba36-32a63ec86239 + 16efc98a-513f-46b6-b107-bf21926b4162: !Template + answer_choices: A ||| B + id: 16efc98a-513f-46b6-b107-bf21926b4162 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -124,11 +97,43 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: better_A_B + reference: '' + 23eccce7-ff42-4827-b717-2a0a6b72da0f: !Template + answer_choices: Yes ||| No + id: 23eccce7-ff42-4827-b717-2a0a6b72da0f + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes + reference: '' + 5e05f264-b2bd-4a11-a3f5-0919b58f88f1: !Template + answer_choices: Yes ||| No + id: 5e05f264-b2bd-4a11-a3f5-0919b58f88f1 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no reference: '' - 85c6d2a7-e090-42e8-be22-eb22350aa724: !Template + 6c0bb4da-2905-4b62-a754-841daefcc9e1: !Template answer_choices: 1 ||| 2 - id: 85c6d2a7-e090-42e8-be22-eb22350aa724 + id: 6c0bb4da-2905-4b62-a754-841daefcc9e1 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -165,29 +170,62 @@ templates: original_task: true name: grammatical_between_1_2 reference: '' - 8e1c6033-6525-410b-b94e-bd7a77b7fe60: !Template + 7446318d-5013-4e9b-a814-e1fcf3fb5136: !Template + answer_choices: Yes ||| No + id: 7446318d-5013-4e9b-a814-e1fcf3fb5136 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + a16ed376-1e5c-43fb-8543-cba234ebdec3: !Template + answer_choices: Yes ||| No + id: a16ed376-1e5c-43fb-8543-cba234ebdec3 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes + reference: '' + bc2abaee-e40f-4a02-9221-7e87263d4fb9: !Template answer_choices: A ||| B - id: 8e1c6033-6525-410b-b94e-bd7a77b7fe60 + id: bc2abaee-e40f-4a02-9221-7e87263d4fb9 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -201,26 +239,26 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: grammatical_which_one_A_B reference: '' - aaca0df7-7c36-4bb2-ba51-fe083881769c: !Template - answer_choices: A ||| B - id: aaca0df7-7c36-4bb2-ba51-fe083881769c + cbeb4680-e03a-4c13-a8e1-c6527d0cec06: !Template + answer_choices: 1 ||| 2 + id: cbeb4680-e03a-4c13-a8e1-c6527d0cec06 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -240,11 +278,11 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: grammatical_which_one_1_2 reference: '' - abe0fa7e-b84f-44b2-91c2-9ba8f64d18c8: !Template + ea9cd9cb-8c42-4f10-aff0-47a76211cb6f: !Template answer_choices: A ||| B - id: abe0fa7e-b84f-44b2-91c2-9ba8f64d18c8 + id: ea9cd9cb-8c42-4f10-aff0-47a76211cb6f jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer A or B. @@ -281,19 +319,3 @@ templates: original_task: true name: grammatical_between_A_B reference: '' - d0cc6ba7-4cfe-458b-acab-3da34a1ef42e: !Template - answer_choices: Yes ||| No - id: d0cc6ba7-4cfe-458b-acab-3da34a1ef42e - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no - reference: '' diff --git a/promptsource/templates/blimp/existential_there_object_raising/templates.yaml b/promptsource/templates/blimp/existential_there_object_raising/templates.yaml index 24b8b3c50..35960a05f 100644 --- a/promptsource/templates/blimp/existential_there_object_raising/templates.yaml +++ b/promptsource/templates/blimp/existential_there_object_raising/templates.yaml @@ -1,9 +1,25 @@ dataset: blimp subset: existential_there_object_raising templates: - 05b0f0ab-68b3-4280-99e6-7d721cfe1d17: !Template + 0678b145-602b-46ce-8229-3e17cec7f729: !Template + answer_choices: Yes ||| No + id: 0678b145-602b-46ce-8229-3e17cec7f729 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 069fc06a-bb2a-45cf-aa15-f5effc0b9992: !Template answer_choices: 1 ||| 2 - id: 05b0f0ab-68b3-4280-99e6-7d721cfe1d17 + id: 069fc06a-bb2a-45cf-aa15-f5effc0b9992 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer 1 or 2. @@ -40,12 +56,12 @@ templates: original_task: true name: grammatical_which_one_1_2 reference: '' - 59e89611-a575-403c-adb4-e703f578f618: !Template + 1b50e808-5621-4d8b-bcc5-a6462e1093f7: !Template answer_choices: A ||| B - id: 59e89611-a575-403c-adb4-e703f578f618 + id: 1b50e808-5621-4d8b-bcc5-a6462e1093f7 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -77,14 +93,30 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: grammatical_which_one_A_B + reference: '' + 4db5a8c2-b6bf-4f7f-98eb-f0e490ee1f4a: !Template + answer_choices: Yes ||| No + id: 4db5a8c2-b6bf-4f7f-98eb-f0e490ee1f4a + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no reference: '' - 5fe1eba0-f56a-4faf-b87e-5bfd67dbfd37: !Template + 51d1e48e-e478-4754-8ad1-d4a94879a859: !Template answer_choices: A ||| B - id: 5fe1eba0-f56a-4faf-b87e-5bfd67dbfd37 + id: 51d1e48e-e478-4754-8ad1-d4a94879a859 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -116,11 +148,11 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: grammatical_between_A_B reference: '' - 62b5bce1-fcc6-475c-a424-c38fa7827dc8: !Template + 5af97eda-fcb2-494a-9ff8-2667b9c45384: !Template answer_choices: Yes ||| No - id: 62b5bce1-fcc6-475c-a424-c38fa7827dc8 + id: 5af97eda-fcb2-494a-9ff8-2667b9c45384 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -134,46 +166,29 @@ templates: original_task: false name: single_sentence_good_no_yes reference: '' - 676499fa-0650-4a98-ace8-b3b10423b719: !Template - answer_choices: Yes ||| No - id: 676499fa-0650-4a98-ace8-b3b10423b719 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no - reference: '' - 69dcb136-2b75-49ea-9256-9033c454f3f3: !Template - answer_choices: 1 ||| 2 - id: 69dcb136-2b75-49ea-9256-9033c454f3f3 + 66fd650e-857b-4630-acb6-35d9c221f5e0: !Template + answer_choices: A ||| B + id: 66fd650e-857b-4630-acb6-35d9c221f5e0 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -187,31 +202,32 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: better_A_B reference: '' - 82f47664-d17b-493a-a009-6ef086ed9510: !Template - answer_choices: A ||| B - id: 82f47664-d17b-493a-a009-6ef086ed9510 + 6b0ad949-f581-4e96-9e49-ea019e34b56d: !Template + answer_choices: 1 ||| 2 + id: 6b0ad949-f581-4e96-9e49-ea019e34b56d jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -225,11 +241,11 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: grammatical_between_1_2 reference: '' - 91a5714e-0948-4930-b6fb-05a3a733f80e: !Template + 7d778e1e-71ce-4c04-81fb-0910f2df8152: !Template answer_choices: A ||| B - id: 91a5714e-0948-4930-b6fb-05a3a733f80e + id: 7d778e1e-71ce-4c04-81fb-0910f2df8152 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -248,7 +264,7 @@ templates: {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order + Which one is a better sentence of English, B or A? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -263,27 +279,33 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: better_B_A reference: '' - a7640552-1d97-4f0f-90ea-5f4002d150f0: !Template - answer_choices: Yes ||| No - id: a7640552-1d97-4f0f-90ea-5f4002d150f0 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + 8502a14a-57fb-4be7-b6ea-cd954e6c07fc: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 8502a14a-57fb-4be7-b6ea-cd954e6c07fc + jinja: '" ||| {{ sentence_good }}' metadata: !TemplateMetadata - choices_in_prompt: true + choices_in_prompt: false metrics: - Accuracy - original_task: false - name: single_sentence_good_yes_no + original_task: true + name: null_prompt + reference: '' + ad3b4869-aedb-4660-9c59-fa2d0aee33bf: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: ad3b4869-aedb-4660-9c59-fa2d0aee33bf + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: true_null_prompt reference: '' - bb421227-948f-47c4-907c-542b98795b65: !Template + fd77c0c2-fc51-482a-94d8-b9923c124146: !Template answer_choices: Yes ||| No - id: bb421227-948f-47c4-907c-542b98795b65 + id: fd77c0c2-fc51-482a-94d8-b9923c124146 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" diff --git a/promptsource/templates/blimp/existential_there_quantifiers_1/templates.yaml b/promptsource/templates/blimp/existential_there_quantifiers_1/templates.yaml index 7552a3d2f..59273a92b 100644 --- a/promptsource/templates/blimp/existential_there_quantifiers_1/templates.yaml +++ b/promptsource/templates/blimp/existential_there_quantifiers_1/templates.yaml @@ -1,9 +1,20 @@ dataset: blimp subset: existential_there_quantifiers_1 templates: - 1360ba62-2cd9-4bd2-ac31-eb0c01d8b746: !Template + 00028abe-efa9-4fe7-a933-5c3669a05f69: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 00028abe-efa9-4fe7-a933-5c3669a05f69 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: true_null_prompt + reference: '' + 2534a971-4e47-417e-8b5f-190690c614b0: !Template answer_choices: A ||| B - id: 1360ba62-2cd9-4bd2-ac31-eb0c01d8b746 + id: 2534a971-4e47-417e-8b5f-190690c614b0 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer A or B. @@ -40,29 +51,30 @@ templates: original_task: true name: grammatical_which_one_A_B reference: '' - 4371ddfa-c506-4428-8a5b-72b93d0c95f2: !Template + 2f5a6a22-a6d8-4bb2-9406-86dc5e51a7ed: !Template answer_choices: A ||| B - id: 4371ddfa-c506-4428-8a5b-72b93d0c95f2 + id: 2f5a6a22-a6d8-4bb2-9406-86dc5e51a7ed jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -76,32 +88,31 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: grammatical_between_A_B reference: '' - 666d69fb-9b69-46ea-9fee-9edb7314a796: !Template + 3d8bd633-92b7-411d-9778-2f6882ed2949: !Template answer_choices: A ||| B - id: 666d69fb-9b69-46ea-9fee-9edb7314a796 + id: 3d8bd633-92b7-411d-9778-2f6882ed2949 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -115,11 +126,11 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: better_A_B reference: '' - 67c2cc5f-809a-4729-b860-15ee05624695: !Template + 4f6eabc7-c283-453f-b027-78d67a66d863: !Template answer_choices: 1 ||| 2 - id: 67c2cc5f-809a-4729-b860-15ee05624695 + id: 4f6eabc7-c283-453f-b027-78d67a66d863 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer 1 or 2. @@ -156,13 +167,13 @@ templates: original_task: true name: grammatical_which_one_1_2 reference: '' - 6fa18a95-644d-4587-8d9a-2c756e816250: !Template + 5055533c-854f-4545-a23d-b296d88475a8: !Template answer_choices: Yes ||| No - id: 6fa18a95-644d-4587-8d9a-2c756e816250 + id: 5055533c-854f-4545-a23d-b296d88475a8 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -170,11 +181,11 @@ templates: metrics: - Accuracy original_task: false - name: single_sentence_good_yes_no + name: single_sentence_good_no_yes reference: '' - 7656ed79-c62c-4f0c-bb5e-4681c27f3f6c: !Template + 5100e29b-d3b1-4440-8075-1d24c5978a00: !Template answer_choices: Yes ||| No - id: 7656ed79-c62c-4f0c-bb5e-4681c27f3f6c + id: 5100e29b-d3b1-4440-8075-1d24c5978a00 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -188,9 +199,52 @@ templates: original_task: false name: single_sentence_bad_yes_no reference: '' - a87f84d3-d52f-4b70-9b2c-a71fb46e3019: !Template + 6d9bd1d3-32ee-4485-8cff-91163197f529: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 6d9bd1d3-32ee-4485-8cff-91163197f529 + jinja: '" ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + 963ef2f1-9270-42b7-9cb5-f83fb23a247b: !Template + answer_choices: Yes ||| No + id: 963ef2f1-9270-42b7-9cb5-f83fb23a247b + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes + reference: '' + 97add579-6c16-4da4-a267-6f34f9bcf3ab: !Template + answer_choices: Yes ||| No + id: 97add579-6c16-4da4-a267-6f34f9bcf3ab + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 9a64d9a6-7f4b-4e08-93ed-fd06a2c9e048: !Template answer_choices: 1 ||| 2 - id: a87f84d3-d52f-4b70-9b2c-a71fb46e3019 + id: 9a64d9a6-7f4b-4e08-93ed-fd06a2c9e048 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -227,25 +281,9 @@ templates: original_task: true name: grammatical_between_1_2 reference: '' - d0e8b05c-5ff1-4e31-b155-e63d55c41cd5: !Template - answer_choices: Yes ||| No - id: d0e8b05c-5ff1-4e31-b155-e63d55c41cd5 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes - reference: '' - e039f4ee-f93b-4572-babf-b1cd29bed2c5: !Template + c2bf5161-5e8f-4c19-b50e-cd19f37892bc: !Template answer_choices: A ||| B - id: e039f4ee-f93b-4572-babf-b1cd29bed2c5 + id: c2bf5161-5e8f-4c19-b50e-cd19f37892bc jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -281,19 +319,3 @@ templates: original_task: true name: better_B_A reference: '' - eae69c1e-3809-46ef-b9b8-5a6eb8a07c1d: !Template - answer_choices: Yes ||| No - id: eae69c1e-3809-46ef-b9b8-5a6eb8a07c1d - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes - reference: '' diff --git a/promptsource/templates/blimp/existential_there_quantifiers_2/templates.yaml b/promptsource/templates/blimp/existential_there_quantifiers_2/templates.yaml index 87d5c6794..35888105e 100644 --- a/promptsource/templates/blimp/existential_there_quantifiers_2/templates.yaml +++ b/promptsource/templates/blimp/existential_there_quantifiers_2/templates.yaml @@ -1,24 +1,24 @@ dataset: blimp subset: existential_there_quantifiers_2 templates: - 344a1e36-e545-4446-a016-519b4b1fe605: !Template - answer_choices: 1 ||| 2 - id: 344a1e36-e545-4446-a016-519b4b1fe605 + 06081764-6071-4b5b-999b-547ea616d8b8: !Template + answer_choices: A ||| B + id: 06081764-6071-4b5b-999b-547ea616d8b8 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -38,11 +38,11 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: grammatical_which_one_A_B reference: '' - 451b7268-d0ea-4e09-a010-b87a59a90a09: !Template + 17a25d1e-bae1-4da3-ba3e-10f44cdecc3e: !Template answer_choices: Yes ||| No - id: 451b7268-d0ea-4e09-a010-b87a59a90a09 + id: 17a25d1e-bae1-4da3-ba3e-10f44cdecc3e jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -56,30 +56,72 @@ templates: original_task: false name: single_sentence_bad_yes_no reference: '' - 4ff8ebc7-c939-4142-bfb7-c5f1d02b1a45: !Template + 5bad950a-d854-4711-953c-b0ccb8657d33: !Template + answer_choices: Yes ||| No + id: 5bad950a-d854-4711-953c-b0ccb8657d33 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes + reference: '' + 5c272bae-e167-4887-a6c6-3e57160e69cb: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 5c272bae-e167-4887-a6c6-3e57160e69cb + jinja: '" ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + 78a211a2-bda9-48e7-b08d-193b43f4702d: !Template + answer_choices: Yes ||| No + id: 78a211a2-bda9-48e7-b08d-193b43f4702d + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes + reference: '' + 9366c196-0b41-40a6-8814-2f83cf06bfa2: !Template answer_choices: A ||| B - id: 4ff8ebc7-c939-4142-bfb7-c5f1d02b1a45 + id: 9366c196-0b41-40a6-8814-2f83cf06bfa2 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -93,32 +135,31 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: better_B_A reference: '' - 888ab7df-512d-416d-ad3a-5a26d3d1f072: !Template + b9018390-cd5e-4542-826a-0548f9c0a3bf: !Template answer_choices: A ||| B - id: 888ab7df-512d-416d-ad3a-5a26d3d1f072 + id: b9018390-cd5e-4542-826a-0548f9c0a3bf jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -132,31 +173,15 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B - reference: '' - 925ec313-6b21-47e5-ba43-c3b530549f40: !Template - answer_choices: Yes ||| No - id: 925ec313-6b21-47e5-ba43-c3b530549f40 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes + name: better_A_B reference: '' - 94d87926-20ed-4e60-936b-7b2be9d498f5: !Template + c5908eee-a950-44e8-a38d-a86df9b8a747: !Template answer_choices: Yes ||| No - id: 94d87926-20ed-4e60-936b-7b2be9d498f5 + id: c5908eee-a950-44e8-a38d-a86df9b8a747 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -164,42 +189,37 @@ templates: metrics: - Accuracy original_task: false - name: single_sentence_good_no_yes + name: single_sentence_good_yes_no reference: '' - d58c25a5-ca52-4adc-874d-a8e80fc71baa: !Template - answer_choices: Yes ||| No - id: d58c25a5-ca52-4adc-874d-a8e80fc71baa - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + c5a2336c-4465-470b-88a8-295856d10426: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: c5a2336c-4465-470b-88a8-295856d10426 + jinja: ' ||| {{ sentence_good }}' metadata: !TemplateMetadata - choices_in_prompt: true + choices_in_prompt: false metrics: - Accuracy - original_task: false - name: single_sentence_good_yes_no + original_task: true + name: true_null_prompt reference: '' - d76b260b-90bc-42de-b23a-dac7449d8e67: !Template - answer_choices: 1 ||| 2 - id: d76b260b-90bc-42de-b23a-dac7449d8e67 + c6b3931a-868a-4e28-9793-b277e810b366: !Template + answer_choices: A ||| B + id: c6b3931a-868a-4e28-9793-b277e810b366 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -219,31 +239,32 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: grammatical_between_A_B reference: '' - ec7c4445-1cc6-4e59-b1f7-40ebb2362670: !Template - answer_choices: A ||| B - id: ec7c4445-1cc6-4e59-b1f7-40ebb2362670 + de509e9e-dd20-47d6-b502-20522765ac74: !Template + answer_choices: 1 ||| 2 + id: de509e9e-dd20-47d6-b502-20522765ac74 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -257,31 +278,32 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: grammatical_which_one_1_2 reference: '' - f074d38b-9b01-4641-8c49-237978c8358a: !Template - answer_choices: A ||| B - id: f074d38b-9b01-4641-8c49-237978c8358a + eccbb1b2-b932-4a52-bd4d-945ff5c8742c: !Template + answer_choices: 1 ||| 2 + id: eccbb1b2-b932-4a52-bd4d-945ff5c8742c jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -295,5 +317,5 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: grammatical_between_1_2 reference: '' diff --git a/promptsource/templates/blimp/existential_there_subject_raising/templates.yaml b/promptsource/templates/blimp/existential_there_subject_raising/templates.yaml index ff24dceae..137b72114 100644 --- a/promptsource/templates/blimp/existential_there_subject_raising/templates.yaml +++ b/promptsource/templates/blimp/existential_there_subject_raising/templates.yaml @@ -1,9 +1,20 @@ dataset: blimp subset: existential_there_subject_raising templates: - 081bd04d-338a-4875-9a3c-0779ff7b58fd: !Template + 386a8e42-071b-450b-a52a-d7e58f298459: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 386a8e42-071b-450b-a52a-d7e58f298459 + jinja: '" ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + 53b18599-30d9-4e99-9d3a-9e6d729a1edd: !Template answer_choices: Yes ||| No - id: 081bd04d-338a-4875-9a3c-0779ff7b58fd + id: 53b18599-30d9-4e99-9d3a-9e6d729a1edd jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -17,51 +28,12 @@ templates: original_task: false name: single_sentence_good_no_yes reference: '' - 100e59a9-57ad-4086-b4e0-a7b401e81439: !Template - answer_choices: A ||| B - id: 100e59a9-57ad-4086-b4e0-a7b401e81439 - jinja: '{% set shuffled_order = [0, 1] | random %} - - Which one of the following sentences is grammatical? Please answer A or B. - - {% if shuffled_order == 0 %} - - A: {{ sentence_good }} - - B: {{ sentence_bad }} - - {% else %} - - A: {{ sentence_bad }} - - B: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} - - {% endif %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: grammatical_which_one_A_B - reference: '' - 29073d22-1639-42fb-b95f-057f42fa6cfe: !Template + 5b565fdc-1f30-48d9-8685-3d0ac35d15cb: !Template answer_choices: 1 ||| 2 - id: 29073d22-1639-42fb-b95f-057f42fa6cfe + id: 5b565fdc-1f30-48d9-8685-3d0ac35d15cb jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -93,31 +65,32 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: grammatical_between_1_2 reference: '' - 5ddd7407-a777-45c4-8db3-3efc77cf80b7: !Template + 67fdfd3f-a5c8-4358-884a-ca3f5ef6552c: !Template answer_choices: A ||| B - id: 5ddd7407-a777-45c4-8db3-3efc77cf80b7 + id: 67fdfd3f-a5c8-4358-884a-ca3f5ef6552c jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -131,27 +104,27 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: grammatical_which_one_A_B reference: '' - 69c565d8-e48a-4ab8-ba9a-c42936192efb: !Template + 82ce18ab-4ba9-4be5-81d6-2f0edf50d743: !Template answer_choices: Yes ||| No - id: 69c565d8-e48a-4ab8-ba9a-c42936192efb + id: 82ce18ab-4ba9-4be5-81d6-2f0edf50d743 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_good_yes_no + name: single_sentence_bad_no_yes reference: '' - ae7f1f09-f621-4adc-8011-6b0ad7ca9bdc: !Template + adaf26f9-8026-488a-b306-513854c0383d: !Template answer_choices: Yes ||| No - id: ae7f1f09-f621-4adc-8011-6b0ad7ca9bdc + id: adaf26f9-8026-488a-b306-513854c0383d jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -165,9 +138,9 @@ templates: original_task: false name: single_sentence_bad_yes_no reference: '' - b01e7768-6ca7-45ee-bc88-e486c616760e: !Template + b5259319-925d-47d9-b47f-7dabf6f8d2e9: !Template answer_choices: A ||| B - id: b01e7768-6ca7-45ee-bc88-e486c616760e + id: b5259319-925d-47d9-b47f-7dabf6f8d2e9 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -203,40 +176,62 @@ templates: original_task: true name: better_A_B reference: '' - c52a261b-fef4-4f3d-8d8e-f8372eec8e0b: !Template - answer_choices: Yes ||| No - id: c52a261b-fef4-4f3d-8d8e-f8372eec8e0b - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + c623eb3d-96f0-4a05-b1d8-f9e02c04510a: !Template + answer_choices: A ||| B + id: c623eb3d-96f0-4a05-b1d8-f9e02c04510a + jinja: '{% set shuffled_order = [0, 1] | random %} - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: false - name: single_sentence_bad_no_yes + original_task: true + name: better_B_A reference: '' - ca577db0-28ac-4b80-8e26-837772f5a8e8: !Template - answer_choices: 1 ||| 2 - id: ca577db0-28ac-4b80-8e26-837772f5a8e8 + c87e10a4-2113-430a-ab3d-179bff143438: !Template + answer_choices: A ||| B + id: c87e10a4-2113-430a-ab3d-179bff143438 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -256,26 +251,53 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: grammatical_between_A_B reference: '' - de92564e-4bfc-4799-95e9-1e5b1693ba53: !Template - answer_choices: A ||| B - id: de92564e-4bfc-4799-95e9-1e5b1693ba53 + d51c69d3-0c0e-409f-b080-093c03dbea5c: !Template + answer_choices: Yes ||| No + id: d51c69d3-0c0e-409f-b080-093c03dbea5c + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + f0a0beca-ddfd-46d9-a6f5-904b90715b72: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: f0a0beca-ddfd-46d9-a6f5-904b90715b72 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: true_null_prompt + reference: '' + fc64b938-2a21-4f32-8d91-b3fc410cbb94: !Template + answer_choices: 1 ||| 2 + id: fc64b938-2a21-4f32-8d91-b3fc410cbb94 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -295,5 +317,5 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: grammatical_which_one_1_2 reference: '' diff --git a/promptsource/templates/blimp/expletive_it_object_raising/templates.yaml b/promptsource/templates/blimp/expletive_it_object_raising/templates.yaml index c0f37c21f..0e27ccebc 100644 --- a/promptsource/templates/blimp/expletive_it_object_raising/templates.yaml +++ b/promptsource/templates/blimp/expletive_it_object_raising/templates.yaml @@ -1,30 +1,45 @@ dataset: blimp subset: expletive_it_object_raising templates: - 2738759e-e8b1-4601-8adb-c8d6d018cc1e: !Template - answer_choices: 1 ||| 2 - id: 2738759e-e8b1-4601-8adb-c8d6d018cc1e + 30dc01cc-0ffd-4863-abf7-91bdf632a1b2: !Template + answer_choices: Yes ||| No + id: 30dc01cc-0ffd-4863-abf7-91bdf632a1b2 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes + reference: '' + 387a7a2a-2226-4b1e-b420-f73f1fa2f659: !Template + answer_choices: A ||| B + id: 387a7a2a-2226-4b1e-b420-f73f1fa2f659 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -38,15 +53,15 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: better_A_B reference: '' - 2b7cee78-c986-43f3-b96f-7410e466b14f: !Template + 5f0000e3-4794-43d6-b141-4df8a99edcea: !Template answer_choices: Yes ||| No - id: 2b7cee78-c986-43f3-b96f-7410e466b14f + id: 5f0000e3-4794-43d6-b141-4df8a99edcea jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -54,26 +69,26 @@ templates: metrics: - Accuracy original_task: false - name: single_sentence_good_no_yes + name: single_sentence_good_yes_no reference: '' - 3a093a67-8a99-4382-b040-84343772a821: !Template - answer_choices: A ||| B - id: 3a093a67-8a99-4382-b040-84343772a821 + 865c9f02-1ce5-44e9-8b65-375cd5a65cec: !Template + answer_choices: 1 ||| 2 + id: 865c9f02-1ce5-44e9-8b65-375cd5a65cec jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -93,11 +108,11 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: grammatical_which_one_1_2 reference: '' - 6f3422f5-2fdc-4747-95c2-1f8c6114e735: !Template + 92d10a40-5945-4fbb-9897-e8537776a7af: !Template answer_choices: Yes ||| No - id: 6f3422f5-2fdc-4747-95c2-1f8c6114e735 + id: 92d10a40-5945-4fbb-9897-e8537776a7af jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -111,24 +126,35 @@ templates: original_task: false name: single_sentence_bad_yes_no reference: '' - 76b00939-25a0-4ca9-aa0c-7ef73f05c922: !Template - answer_choices: A ||| B - id: 76b00939-25a0-4ca9-aa0c-7ef73f05c922 + b19ed231-6114-418d-b09c-834d28a5d982: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: b19ed231-6114-418d-b09c-834d28a5d982 + jinja: '" ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + bd54d076-5b93-43a7-9322-45fdb93206d4: !Template + answer_choices: 1 ||| 2 + id: bd54d076-5b93-43a7-9322-45fdb93206d4 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -148,11 +174,11 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: grammatical_between_1_2 reference: '' - 7c662985-1117-43a8-a085-89fb33695c32: !Template + c30d9409-67bd-4bd2-9366-becbd4b961ed: !Template answer_choices: A ||| B - id: 7c662985-1117-43a8-a085-89fb33695c32 + id: c30d9409-67bd-4bd2-9366-becbd4b961ed jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -188,56 +214,24 @@ templates: original_task: true name: better_B_A reference: '' - 9bda7c86-3f6c-45d8-a5ce-29a79a68bb4b: !Template - answer_choices: Yes ||| No - id: 9bda7c86-3f6c-45d8-a5ce-29a79a68bb4b - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes - reference: '' - a38b698a-268c-49e8-9a6f-3eea14c6daef: !Template - answer_choices: Yes ||| No - id: a38b698a-268c-49e8-9a6f-3eea14c6daef - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no - reference: '' - a59375ab-3c07-4678-8b1c-53acb31357ef: !Template - answer_choices: 1 ||| 2 - id: a59375ab-3c07-4678-8b1c-53acb31357ef + cbe88ae3-3d99-4e72-b99b-271192c57627: !Template + answer_choices: A ||| B + id: cbe88ae3-3d99-4e72-b99b-271192c57627 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -257,31 +251,59 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: grammatical_between_A_B reference: '' - cf136e9f-e1f6-47f4-91c2-4ca1992a22c9: !Template + d2816a2d-58f2-49b3-a8e0-6a19ec1babfc: !Template + answer_choices: Yes ||| No + id: d2816a2d-58f2-49b3-a8e0-6a19ec1babfc + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes + reference: '' + e18fdda4-907a-4ea6-8865-04e153d880a5: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: e18fdda4-907a-4ea6-8865-04e153d880a5 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: true_null_prompt + reference: '' + ef751a04-c17f-4d8b-8c0e-080d58475c96: !Template answer_choices: A ||| B - id: cf136e9f-e1f6-47f4-91c2-4ca1992a22c9 + id: ef751a04-c17f-4d8b-8c0e-080d58475c96 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -295,5 +317,5 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: grammatical_which_one_A_B reference: '' diff --git a/promptsource/templates/blimp/inchoative/templates.yaml b/promptsource/templates/blimp/inchoative/templates.yaml index 499d26916..a9cedde75 100644 --- a/promptsource/templates/blimp/inchoative/templates.yaml +++ b/promptsource/templates/blimp/inchoative/templates.yaml @@ -1,29 +1,57 @@ dataset: blimp subset: inchoative templates: - 218b06da-2d74-429d-af75-97a69d9a2919: !Template + 01c547f6-f10c-4cf5-9294-68aaf09487f8: !Template + answer_choices: Yes ||| No + id: 01c547f6-f10c-4cf5-9294-68aaf09487f8 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes + reference: '' + 064910ab-4d4c-4033-9c1a-4dfd9d177d70: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 064910ab-4d4c-4033-9c1a-4dfd9d177d70 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: true_null_prompt + reference: '' + 25da318f-ebd9-4167-b2eb-ddc5284b4371: !Template answer_choices: A ||| B - id: 218b06da-2d74-429d-af75-97a69d9a2919 + id: 25da318f-ebd9-4167-b2eb-ddc5284b4371 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -37,14 +65,14 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: grammatical_which_one_A_B reference: '' - 58ec605f-984b-4907-bf95-7fa933b2bffb: !Template + 3e5919ff-4762-48be-9468-55116d6234fe: !Template answer_choices: 1 ||| 2 - id: 58ec605f-984b-4907-bf95-7fa933b2bffb + id: 3e5919ff-4762-48be-9468-55116d6234fe jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -76,62 +104,14 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 - reference: '' - 67f86c5c-16e5-47be-bd77-5cd471b7c917: !Template - answer_choices: Yes ||| No - id: 67f86c5c-16e5-47be-bd77-5cd471b7c917 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes - reference: '' - 68239fd8-a74e-40ff-b5a5-7767bcd67ff2: !Template - answer_choices: Yes ||| No - id: 68239fd8-a74e-40ff-b5a5-7767bcd67ff2 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no - reference: '' - 6a940bc4-fb9e-4716-b7a0-9f0617c473f5: !Template - answer_choices: Yes ||| No - id: 6a940bc4-fb9e-4716-b7a0-9f0617c473f5 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes + name: grammatical_which_one_1_2 reference: '' - 74826ac7-9337-4a80-9924-6d3b84565729: !Template + 4837edeb-7505-4db8-8465-0d81c89b1b1d: !Template answer_choices: A ||| B - id: 74826ac7-9337-4a80-9924-6d3b84565729 + id: 4837edeb-7505-4db8-8465-0d81c89b1b1d jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -163,11 +143,11 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: grammatical_between_A_B reference: '' - 93166c80-802e-4c35-a340-f75b9ec7c80a: !Template + 4dc9d016-50f8-4c7e-9329-4adaeeb9abc6: !Template answer_choices: A ||| B - id: 93166c80-802e-4c35-a340-f75b9ec7c80a + id: 4dc9d016-50f8-4c7e-9329-4adaeeb9abc6 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -203,13 +183,24 @@ templates: original_task: true name: better_A_B reference: '' - b543de02-39cf-4086-bd21-b45ad43211ec: !Template + 59b356d9-53ca-491c-820f-e7b2b7eb1274: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 59b356d9-53ca-491c-820f-e7b2b7eb1274 + jinja: '" ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + 6a549a6e-f2da-48f7-b565-82fe320c7067: !Template answer_choices: Yes ||| No - id: b543de02-39cf-4086-bd21-b45ad43211ec + id: 6a549a6e-f2da-48f7-b565-82fe320c7067 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata @@ -217,14 +208,14 @@ templates: metrics: - Accuracy original_task: false - name: single_sentence_bad_yes_no + name: single_sentence_bad_no_yes reference: '' - be72e247-5f10-4be1-a938-54bd469f443a: !Template + 6f77d0b0-4def-44cb-8989-94cf63ec1302: !Template answer_choices: 1 ||| 2 - id: be72e247-5f10-4be1-a938-54bd469f443a + id: 6f77d0b0-4def-44cb-8989-94cf63ec1302 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -256,32 +247,63 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: grammatical_between_1_2 + reference: '' + 7198da05-87f2-4fb6-954e-317f6951c56d: !Template + answer_choices: Yes ||| No + id: 7198da05-87f2-4fb6-954e-317f6951c56d + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 724338e9-04f1-494c-b056-498855abcfee: !Template + answer_choices: Yes ||| No + id: 724338e9-04f1-494c-b056-498855abcfee + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no reference: '' - c8a0d6a0-0245-4471-b7bd-83d9159749f8: !Template + c28c1ceb-77b8-43b4-ba2f-6f078f0a3f30: !Template answer_choices: A ||| B - id: c8a0d6a0-0245-4471-b7bd-83d9159749f8 + id: c28c1ceb-77b8-43b4-ba2f-6f078f0a3f30 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -295,5 +317,5 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: better_B_A reference: '' diff --git a/promptsource/templates/blimp/intransitive/templates.yaml b/promptsource/templates/blimp/intransitive/templates.yaml index 57a6795f1..f06f8a0a3 100644 --- a/promptsource/templates/blimp/intransitive/templates.yaml +++ b/promptsource/templates/blimp/intransitive/templates.yaml @@ -1,9 +1,9 @@ dataset: blimp subset: intransitive templates: - 05fe66d3-f4bf-4987-bbdd-84e087faef22: !Template + 3c912318-b915-4cd4-a921-294092393382: !Template answer_choices: A ||| B - id: 05fe66d3-f4bf-4987-bbdd-84e087faef22 + id: 3c912318-b915-4cd4-a921-294092393382 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -22,7 +22,7 @@ templates: {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order + Which one is a better sentence of English, A or B? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -37,42 +37,37 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: better_A_B reference: '' - 135732f1-5ae4-4562-8bd2-c79e326c61f4: !Template - answer_choices: Yes ||| No - id: 135732f1-5ae4-4562-8bd2-c79e326c61f4 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' + 4b3be9c6-383a-4be9-8caf-df394a404632: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 4b3be9c6-383a-4be9-8caf-df394a404632 + jinja: '" ||| {{ sentence_good }}' metadata: !TemplateMetadata - choices_in_prompt: true + choices_in_prompt: false metrics: - Accuracy - original_task: false - name: single_sentence_bad_yes_no + original_task: true + name: null_prompt reference: '' - 1eefc31e-fc87-4284-8a2c-5c6d5c056099: !Template - answer_choices: A ||| B - id: 1eefc31e-fc87-4284-8a2c-5c6d5c056099 + 4d8e4dd3-fe6f-4450-ab3e-ca775388ff6c: !Template + answer_choices: 1 ||| 2 + id: 4d8e4dd3-fe6f-4450-ab3e-ca775388ff6c jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -92,27 +87,11 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B - reference: '' - 3512c0b1-cbec-4c98-9b6b-93632d24ff5a: !Template - answer_choices: Yes ||| No - id: 3512c0b1-cbec-4c98-9b6b-93632d24ff5a - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes + name: grammatical_between_1_2 reference: '' - 5b2f3a36-4482-4000-8a18-983177b4b79b: !Template + 6ebec8d2-e5c0-4f76-bb46-3665befc5dc4: !Template answer_choices: 1 ||| 2 - id: 5b2f3a36-4482-4000-8a18-983177b4b79b + id: 6ebec8d2-e5c0-4f76-bb46-3665befc5dc4 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer 1 or 2. @@ -149,12 +128,12 @@ templates: original_task: true name: grammatical_which_one_1_2 reference: '' - 69bc50bd-327b-471d-9f6f-9e4c9054b119: !Template + 87a67a6f-f043-4d6c-9c51-7ac61df3b55a: !Template answer_choices: A ||| B - id: 69bc50bd-327b-471d-9f6f-9e4c9054b119 + id: 87a67a6f-f043-4d6c-9c51-7ac61df3b55a jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -186,31 +165,32 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: grammatical_which_one_A_B reference: '' - 8c673b2a-62f1-40d5-b11c-4e90e1fdfa00: !Template + a67f5769-edcd-45e7-a930-7ea6849c2e2e: !Template answer_choices: A ||| B - id: 8c673b2a-62f1-40d5-b11c-4e90e1fdfa00 + id: a67f5769-edcd-45e7-a930-7ea6849c2e2e jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -224,32 +204,74 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: grammatical_between_A_B reference: '' - a6a13106-9d4d-45b4-8d4b-cd6749bb9436: !Template - answer_choices: 1 ||| 2 - id: a6a13106-9d4d-45b4-8d4b-cd6749bb9436 + b4069486-a61e-41b4-b4f6-1524ff686a66: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: b4069486-a61e-41b4-b4f6-1524ff686a66 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: true_null_prompt + reference: '' + c7b062bf-7ed7-40c4-84ad-1ec9e3ef51a7: !Template + answer_choices: Yes ||| No + id: c7b062bf-7ed7-40c4-84ad-1ec9e3ef51a7 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + d2f641f2-1a2a-474c-85c5-1d8d49f4a103: !Template + answer_choices: Yes ||| No + id: d2f641f2-1a2a-474c-85c5-1d8d49f4a103 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes + reference: '' + dbb33462-77f8-4e6c-947d-0cabd5f9920f: !Template + answer_choices: A ||| B + id: dbb33462-77f8-4e6c-947d-0cabd5f9920f jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -263,31 +285,31 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: better_B_A reference: '' - abf0bad1-be17-4f0c-bfdc-1983155e6e0c: !Template + e116f131-9021-4f6d-8176-38b58f19058c: !Template answer_choices: Yes ||| No - id: abf0bad1-be17-4f0c-bfdc-1983155e6e0c + id: e116f131-9021-4f6d-8176-38b58f19058c jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_good_yes_no + name: single_sentence_bad_no_yes reference: '' - efb1827e-a6d7-48bf-82cf-1550f11a48c3: !Template + fba1ef34-6f4e-4f19-9bd4-7b01181e2e4b: !Template answer_choices: Yes ||| No - id: efb1827e-a6d7-48bf-82cf-1550f11a48c3 + id: fba1ef34-6f4e-4f19-9bd4-7b01181e2e4b jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -295,5 +317,5 @@ templates: metrics: - Accuracy original_task: false - name: single_sentence_good_no_yes + name: single_sentence_good_yes_no reference: '' diff --git a/promptsource/templates/blimp/irregular_past_participle_adjectives/templates.yaml b/promptsource/templates/blimp/irregular_past_participle_adjectives/templates.yaml index 05aa04fab..2261dd41c 100644 --- a/promptsource/templates/blimp/irregular_past_participle_adjectives/templates.yaml +++ b/promptsource/templates/blimp/irregular_past_participle_adjectives/templates.yaml @@ -1,12 +1,12 @@ dataset: blimp subset: irregular_past_participle_adjectives templates: - 01e46cac-569b-43e6-9bc0-569fc9f00fbd: !Template + 16a8523d-2f5c-4d89-aedf-6f2a79a9bfec: !Template answer_choices: A ||| B - id: 01e46cac-569b-43e6-9bc0-569fc9f00fbd + id: 16a8523d-2f5c-4d89-aedf-6f2a79a9bfec jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -38,11 +38,27 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: grammatical_which_one_A_B + reference: '' + 1d494a1b-ee23-4ffb-bcfb-4e1a160bf245: !Template + answer_choices: Yes ||| No + id: 1d494a1b-ee23-4ffb-bcfb-4e1a160bf245 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes reference: '' - 654feffd-c577-4f2c-a54c-4e17cc33a3a2: !Template + 2b880aa1-25b4-4694-8ca9-efd89b51e86a: !Template answer_choices: 1 ||| 2 - id: 654feffd-c577-4f2c-a54c-4e17cc33a3a2 + id: 2b880aa1-25b4-4694-8ca9-efd89b51e86a jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -79,80 +95,63 @@ templates: original_task: true name: grammatical_between_1_2 reference: '' - 97bb0e08-0820-4b56-81a9-c206264f4197: !Template - answer_choices: A ||| B - id: 97bb0e08-0820-4b56-81a9-c206264f4197 - jinja: '{% set shuffled_order = [0, 1] | random %} - - Which one of the following sentences is grammatical? Please answer A or B. - - {% if shuffled_order == 0 %} - - A: {{ sentence_good }} - - B: {{ sentence_bad }} - - {% else %} - - A: {{ sentence_bad }} - - B: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} - - {% endif %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: grammatical_which_one_A_B - reference: '' - b3328790-7d90-4dfb-9cf0-b2ff946346e1: !Template + 60e8eab4-8c21-4ef4-9141-196e2c89be04: !Template answer_choices: Yes ||| No - id: b3328790-7d90-4dfb-9cf0-b2ff946346e1 + id: 60e8eab4-8c21-4ef4-9141-196e2c89be04 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_good_no_yes + name: single_sentence_bad_yes_no reference: '' - b70d30d9-77dd-455f-b138-82eb05beff7d: !Template + 68594837-019a-4625-8cc3-bc00e853e757: !Template answer_choices: Yes ||| No - id: b70d30d9-77dd-455f-b138-82eb05beff7d + id: 68594837-019a-4625-8cc3-bc00e853e757 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_good_yes_no + name: single_sentence_bad_no_yes + reference: '' + 7465ee8b-594f-43d7-afc1-a029b99cec06: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 7465ee8b-594f-43d7-afc1-a029b99cec06 + jinja: '" ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt reference: '' - b8ca2315-d995-469c-a25d-5bcf20ed84a4: !Template + 7c16eb5f-05b6-4e73-9fb6-16f91ad1fbb1: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 7c16eb5f-05b6-4e73-9fb6-16f91ad1fbb1 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: true_null_prompt + reference: '' + 87516bcf-34f6-4fde-8142-e9340a54e32c: !Template answer_choices: A ||| B - id: b8ca2315-d995-469c-a25d-5bcf20ed84a4 + id: 87516bcf-34f6-4fde-8142-e9340a54e32c jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -188,41 +187,63 @@ templates: original_task: true name: better_A_B reference: '' - c6c7352a-5b68-4ca8-9db2-70902434b97e: !Template - answer_choices: Yes ||| No - id: c6c7352a-5b68-4ca8-9db2-70902434b97e - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + ca78195f-5b74-4046-9c13-3c1ce1b0c663: !Template + answer_choices: A ||| B + id: ca78195f-5b74-4046-9c13-3c1ce1b0c663 + jinja: '{% set shuffled_order = [0, 1] | random %} - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: false - name: single_sentence_bad_no_yes + original_task: true + name: better_B_A reference: '' - d59d4523-ea2b-4780-8a76-4ddf27f69ca6: !Template + e4c06619-699c-4a12-a3d7-78cc606c199b: !Template answer_choices: Yes ||| No - id: d59d4523-ea2b-4780-8a76-4ddf27f69ca6 + id: e4c06619-699c-4a12-a3d7-78cc606c199b jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_bad_yes_no + name: single_sentence_good_yes_no reference: '' - ef8fe88c-c95f-4b97-9d09-0184c43f4244: !Template + e5daf178-7934-405c-9c83-35aeedc95fed: !Template answer_choices: 1 ||| 2 - id: ef8fe88c-c95f-4b97-9d09-0184c43f4244 + id: e5daf178-7934-405c-9c83-35aeedc95fed jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer 1 or 2. @@ -259,29 +280,30 @@ templates: original_task: true name: grammatical_which_one_1_2 reference: '' - ff43ecaf-279d-41a6-8519-d0c57b5d61ce: !Template + e917d2a5-66ae-4faf-be0d-1f9ad8201d7c: !Template answer_choices: A ||| B - id: ff43ecaf-279d-41a6-8519-d0c57b5d61ce + id: e917d2a5-66ae-4faf-be0d-1f9ad8201d7c jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -295,5 +317,5 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: grammatical_between_A_B reference: '' diff --git a/promptsource/templates/blimp/irregular_past_participle_verbs/templates.yaml b/promptsource/templates/blimp/irregular_past_participle_verbs/templates.yaml index 79a56bb20..9b2df8d9c 100644 --- a/promptsource/templates/blimp/irregular_past_participle_verbs/templates.yaml +++ b/promptsource/templates/blimp/irregular_past_participle_verbs/templates.yaml @@ -1,29 +1,46 @@ dataset: blimp subset: irregular_past_participle_verbs templates: - 05aea489-316f-42e0-9868-e6e516fea9b0: !Template - answer_choices: A ||| B - id: 05aea489-316f-42e0-9868-e6e516fea9b0 + 01795e7b-f8b1-45a0-8ae1-f1c0e3b21adb: !Template + answer_choices: Yes ||| No + id: 01795e7b-f8b1-45a0-8ae1-f1c0e3b21adb + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes + reference: '' + 048a9ad5-6cd1-4927-a9c2-3682d1c3dfe6: !Template + answer_choices: 1 ||| 2 + id: 048a9ad5-6cd1-4927-a9c2-3682d1c3dfe6 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -37,14 +54,46 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: grammatical_between_1_2 + reference: '' + 3a6f271c-7d0c-4586-b475-54c32ce98e5b: !Template + answer_choices: Yes ||| No + id: 3a6f271c-7d0c-4586-b475-54c32ce98e5b + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no reference: '' - 2d2b924b-b765-41f6-8d6d-e8f6238d1abf: !Template + 79827cfd-939e-4bed-938a-c511e4e74446: !Template + answer_choices: Yes ||| No + id: 79827cfd-939e-4bed-938a-c511e4e74446 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes + reference: '' + 7a1e655c-f284-4966-a5f3-df64f9ec6ba0: !Template answer_choices: A ||| B - id: 2d2b924b-b765-41f6-8d6d-e8f6238d1abf + id: 7a1e655c-f284-4966-a5f3-df64f9ec6ba0 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -76,32 +125,69 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: grammatical_between_A_B + reference: '' + 89318530-c50e-43ba-a7fc-528a8b23f8c1: !Template + answer_choices: Yes ||| No + id: 89318530-c50e-43ba-a7fc-528a8b23f8c1 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no reference: '' - 464409e0-a6b0-4b72-b39b-e4d19557bec5: !Template + 92fc33a6-dec5-44a2-b12a-1550ebaaac17: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 92fc33a6-dec5-44a2-b12a-1550ebaaac17 + jinja: '" ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + 9aecc761-2e55-4553-afae-ead346c4e6b4: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 9aecc761-2e55-4553-afae-ead346c4e6b4 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: true_null_prompt + reference: '' + c9ebec28-a6fd-4743-b320-4bafffc2de86: !Template answer_choices: A ||| B - id: 464409e0-a6b0-4b72-b39b-e4d19557bec5 + id: c9ebec28-a6fd-4743-b320-4bafffc2de86 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -115,26 +201,26 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: better_B_A reference: '' - 6b8e17de-cd52-4744-aa19-05502547c983: !Template - answer_choices: 1 ||| 2 - id: 6b8e17de-cd52-4744-aa19-05502547c983 + cb56e5d4-2628-4c31-9b73-fb7cdb078cc5: !Template + answer_choices: A ||| B + id: cb56e5d4-2628-4c31-9b73-fb7cdb078cc5 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -154,27 +240,11 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 - reference: '' - 719d40ac-f8b6-4b71-a31b-600e493f6ee9: !Template - answer_choices: Yes ||| No - id: 719d40ac-f8b6-4b71-a31b-600e493f6ee9 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no + name: grammatical_which_one_A_B reference: '' - 73c02ed4-3d51-4394-9276-793d54d870a4: !Template + d6ce7716-6632-45dc-b9ae-3395d6f8e728: !Template answer_choices: A ||| B - id: 73c02ed4-3d51-4394-9276-793d54d870a4 + id: d6ce7716-6632-45dc-b9ae-3395d6f8e728 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -193,7 +263,7 @@ templates: {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order + Which one is a better sentence of English, A or B? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -208,30 +278,14 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A - reference: '' - 7e364fac-a76d-47d9-b660-a235be6ad22a: !Template - answer_choices: Yes ||| No - id: 7e364fac-a76d-47d9-b660-a235be6ad22a - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes + name: better_A_B reference: '' - 8342400f-5e9d-4867-bfae-59edecab513f: !Template + ff56058b-27f8-4c79-b1d5-89c56b9c1cf5: !Template answer_choices: 1 ||| 2 - id: 8342400f-5e9d-4867-bfae-59edecab513f + id: ff56058b-27f8-4c79-b1d5-89c56b9c1cf5 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -263,37 +317,5 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 - reference: '' - 94c81ab8-a021-4594-9e67-71b9a35154ec: !Template - answer_choices: Yes ||| No - id: 94c81ab8-a021-4594-9e67-71b9a35154ec - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes - reference: '' - d33b96d4-10ec-4778-aedc-e37d911cfd7f: !Template - answer_choices: Yes ||| No - id: d33b96d4-10ec-4778-aedc-e37d911cfd7f - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no + name: grammatical_which_one_1_2 reference: '' diff --git a/promptsource/templates/blimp/irregular_plural_subject_verb_agreement_1/templates.yaml b/promptsource/templates/blimp/irregular_plural_subject_verb_agreement_1/templates.yaml index 2955fb531..d4916b2e5 100644 --- a/promptsource/templates/blimp/irregular_plural_subject_verb_agreement_1/templates.yaml +++ b/promptsource/templates/blimp/irregular_plural_subject_verb_agreement_1/templates.yaml @@ -1,47 +1,36 @@ dataset: blimp subset: irregular_plural_subject_verb_agreement_1 templates: - 0b237ec0-4979-4207-b2da-b2ea4a51df1e: !Template - answer_choices: A ||| B - id: 0b237ec0-4979-4207-b2da-b2ea4a51df1e - jinja: '{% set shuffled_order = [0, 1] | random %} - - Read the following two sentences. - - {% if shuffled_order == 0 %} - - Sentence A: {{ sentence_good }} - - Sentence B: {{ sentence_bad }} - - {% else %} - - Sentence A: {{ sentence_bad }} - - Sentence B: {{ sentence_good }} - - {% endif %} - - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} + 2a951848-458f-4dad-ab47-f8eac9a74d0a: !Template + answer_choices: Yes ||| No + id: 2a951848-458f-4dad-ab47-f8eac9a74d0a + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - {% endif %}' + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 3a753455-5153-4bb3-8249-66c359466110: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 3a753455-5153-4bb3-8249-66c359466110 + jinja: '" ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy original_task: true - name: better_A_B + name: null_prompt reference: '' - 51be22a7-cfd5-47f0-84b2-fe4c4cf73adb: !Template + 42da8e53-8363-48d0-b793-6922ad672ba0: !Template answer_choices: A ||| B - id: 51be22a7-cfd5-47f0-84b2-fe4c4cf73adb + id: 42da8e53-8363-48d0-b793-6922ad672ba0 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer A or B. @@ -78,24 +67,35 @@ templates: original_task: true name: grammatical_between_A_B reference: '' - 632d0ed3-9a42-4b09-9895-e92e5abf51fa: !Template - answer_choices: 1 ||| 2 - id: 632d0ed3-9a42-4b09-9895-e92e5abf51fa + 7596b6e3-9265-4cd3-bb43-49555416a253: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 7596b6e3-9265-4cd3-bb43-49555416a253 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: true_null_prompt + reference: '' + 77575034-c0b0-42e4-8423-8e8543085834: !Template + answer_choices: A ||| B + id: 77575034-c0b0-42e4-8423-8e8543085834 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -115,31 +115,48 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: grammatical_which_one_A_B reference: '' - 666b1056-86bc-4eac-b193-c47308692e0c: !Template - answer_choices: A ||| B - id: 666b1056-86bc-4eac-b193-c47308692e0c + 7c15530c-1869-428a-975b-46c052055b62: !Template + answer_choices: Yes ||| No + id: 7c15530c-1869-428a-975b-46c052055b62 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes + reference: '' + 8061c884-5f43-4549-a3e4-c1acb736931b: !Template + answer_choices: 1 ||| 2 + id: 8061c884-5f43-4549-a3e4-c1acb736931b jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -153,27 +170,27 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: grammatical_between_1_2 reference: '' - 6be39d40-ddc8-4f8d-8b38-1439d2a0c897: !Template + 80fc5d64-dbac-46af-b42a-af73349215f5: !Template answer_choices: Yes ||| No - id: 6be39d40-ddc8-4f8d-8b38-1439d2a0c897 + id: 80fc5d64-dbac-46af-b42a-af73349215f5 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_good_no_yes + name: single_sentence_bad_yes_no reference: '' - 83c0cac5-5ee1-4fa2-a7f9-6b2e450de34c: !Template + 8c24a79f-adc4-4ac7-9ff4-b73f0b08dcde: !Template answer_choices: Yes ||| No - id: 83c0cac5-5ee1-4fa2-a7f9-6b2e450de34c + id: 8c24a79f-adc4-4ac7-9ff4-b73f0b08dcde jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -187,9 +204,9 @@ templates: original_task: false name: single_sentence_bad_no_yes reference: '' - 8e84f8d8-a18c-44a7-bc14-78b9fdc3e6ca: !Template + 8d93abf0-ef5f-4260-b3cb-eccaabcfb3b5: !Template answer_choices: 1 ||| 2 - id: 8e84f8d8-a18c-44a7-bc14-78b9fdc3e6ca + id: 8d93abf0-ef5f-4260-b3cb-eccaabcfb3b5 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer 1 or 2. @@ -226,46 +243,29 @@ templates: original_task: true name: grammatical_which_one_1_2 reference: '' - 9e3eb092-c65b-452e-989d-b6b9a1dd0617: !Template - answer_choices: Yes ||| No - id: 9e3eb092-c65b-452e-989d-b6b9a1dd0617 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no - reference: '' - b8b3dd57-20b7-4d5e-9b16-64cca88f5502: !Template + a6d2c08b-d883-4d8c-b1de-c47412ffc5a7: !Template answer_choices: A ||| B - id: b8b3dd57-20b7-4d5e-9b16-64cca88f5502 + id: a6d2c08b-d883-4d8c-b1de-c47412ffc5a7 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -279,21 +279,43 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: better_B_A reference: '' - ec92dd7e-d41f-456e-b3d7-78801bd84c54: !Template - answer_choices: Yes ||| No - id: ec92dd7e-d41f-456e-b3d7-78801bd84c54 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + d5548e5c-998b-4f60-96c4-be23b9ace2e0: !Template + answer_choices: A ||| B + id: d5548e5c-998b-4f60-96c4-be23b9ace2e0 + jinja: '{% set shuffled_order = [0, 1] | random %} - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: false - name: single_sentence_bad_yes_no + original_task: true + name: better_A_B reference: '' diff --git a/promptsource/templates/blimp/irregular_plural_subject_verb_agreement_2/templates.yaml b/promptsource/templates/blimp/irregular_plural_subject_verb_agreement_2/templates.yaml index 930746773..bfb288562 100644 --- a/promptsource/templates/blimp/irregular_plural_subject_verb_agreement_2/templates.yaml +++ b/promptsource/templates/blimp/irregular_plural_subject_verb_agreement_2/templates.yaml @@ -1,9 +1,41 @@ dataset: blimp subset: irregular_plural_subject_verb_agreement_2 templates: - 1131584f-b755-4041-8632-edd6be1d49ec: !Template + 4bc70b39-ec97-40a0-aa08-e8323cff9367: !Template + answer_choices: Yes ||| No + id: 4bc70b39-ec97-40a0-aa08-e8323cff9367 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 54a29275-7b43-4af6-8bae-dcb2d51d0e6c: !Template + answer_choices: Yes ||| No + id: 54a29275-7b43-4af6-8bae-dcb2d51d0e6c + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes + reference: '' + 5bfd948d-2a9e-447b-954f-ee7c04daa5f8: !Template answer_choices: A ||| B - id: 1131584f-b755-4041-8632-edd6be1d49ec + id: 5bfd948d-2a9e-447b-954f-ee7c04daa5f8 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -39,25 +71,59 @@ templates: original_task: true name: better_A_B reference: '' - 17692107-00aa-4a55-83f2-0385580c64d5: !Template - answer_choices: Yes ||| No - id: 17692107-00aa-4a55-83f2-0385580c64d5 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + 632d0460-942a-4d0f-970f-2a6e22fda4f6: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 632d0460-942a-4d0f-970f-2a6e22fda4f6 + jinja: '" ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + 8ad81c4e-36a5-481f-be12-7283effb4e05: !Template + answer_choices: 1 ||| 2 + id: 8ad81c4e-36a5-481f-be12-7283effb4e05 + jinja: '{% set shuffled_order = [0, 1] | random %} - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' + Between the two sentences, which one is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: false - name: single_sentence_bad_yes_no + original_task: true + name: grammatical_between_1_2 reference: '' - 2df1de79-e0b7-41d8-a6e8-ff46a3b09769: !Template + 8fbb9f54-2881-4e5c-b83f-8827a288d5db: !Template answer_choices: A ||| B - id: 2df1de79-e0b7-41d8-a6e8-ff46a3b09769 + id: 8fbb9f54-2881-4e5c-b83f-8827a288d5db jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -93,41 +159,9 @@ templates: original_task: true name: better_B_A reference: '' - 39bb2c89-ec48-49ca-8cf2-0a4b9811f177: !Template - answer_choices: Yes ||| No - id: 39bb2c89-ec48-49ca-8cf2-0a4b9811f177 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes - reference: '' - 5828fcad-f8b2-4975-ac98-b363913a6cf0: !Template - answer_choices: Yes ||| No - id: 5828fcad-f8b2-4975-ac98-b363913a6cf0 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes - reference: '' - 770aa1ee-31cf-405e-b2f3-568b651d8ea1: !Template + a77827bc-d9e1-4003-aba0-e3c7a3b86f1c: !Template answer_choices: 1 ||| 2 - id: 770aa1ee-31cf-405e-b2f3-568b651d8ea1 + id: a77827bc-d9e1-4003-aba0-e3c7a3b86f1c jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer 1 or 2. @@ -164,24 +198,24 @@ templates: original_task: true name: grammatical_which_one_1_2 reference: '' - 85283761-2238-492e-ab39-db0992b88cbc: !Template - answer_choices: 1 ||| 2 - id: 85283761-2238-492e-ab39-db0992b88cbc + b9ab8131-0d86-448e-8a84-dc102b61de33: !Template + answer_choices: A ||| B + id: b9ab8131-0d86-448e-8a84-dc102b61de33 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -201,11 +235,11 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: grammatical_between_A_B reference: '' - 9cff0cd3-9611-4169-b3d6-3ea707dbb57c: !Template + d4d5d236-eb1a-449e-a693-3443ca550070: !Template answer_choices: A ||| B - id: 9cff0cd3-9611-4169-b3d6-3ea707dbb57c + id: d4d5d236-eb1a-449e-a693-3443ca550070 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer A or B. @@ -242,9 +276,9 @@ templates: original_task: true name: grammatical_which_one_A_B reference: '' - a51daee4-1098-4141-85fc-4887e8eb506f: !Template + dc551883-e804-49c7-bcc1-d9debef56b5d: !Template answer_choices: Yes ||| No - id: a51daee4-1098-4141-85fc-4887e8eb506f + id: dc551883-e804-49c7-bcc1-d9debef56b5d jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -258,42 +292,30 @@ templates: original_task: false name: single_sentence_good_yes_no reference: '' - e161cd15-e148-4a02-8a31-2ec90546fb05: !Template - answer_choices: A ||| B - id: e161cd15-e148-4a02-8a31-2ec90546fb05 - jinja: '{% set shuffled_order = [0, 1] | random %} - - Between the two sentences, which one is grammatical? Please answer A or B. - - {% if shuffled_order == 0 %} - - A: {{ sentence_good }} - - B: {{ sentence_bad }} - - {% else %} - - A: {{ sentence_bad }} - - B: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} + f2c4adf3-85af-4e31-9656-313fdc77287e: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: f2c4adf3-85af-4e31-9656-313fdc77287e + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: true_null_prompt + reference: '' + f9647cd9-b31f-4d72-a741-96b8e8374458: !Template + answer_choices: Yes ||| No + id: f9647cd9-b31f-4d72-a741-96b8e8374458 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - {% endif %}' + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: grammatical_between_A_B + original_task: false + name: single_sentence_good_no_yes reference: '' diff --git a/promptsource/templates/blimp/left_branch_island_echo_question/templates.yaml b/promptsource/templates/blimp/left_branch_island_echo_question/templates.yaml index c3fa4a49d..eade59b46 100644 --- a/promptsource/templates/blimp/left_branch_island_echo_question/templates.yaml +++ b/promptsource/templates/blimp/left_branch_island_echo_question/templates.yaml @@ -1,24 +1,105 @@ dataset: blimp subset: left_branch_island_echo_question templates: - 00139aeb-52d3-47c1-a2fe-6b1e94f37f01: !Template + 3a8adcf5-1de5-44df-86dc-5d4180aed887: !Template answer_choices: A ||| B - id: 00139aeb-52d3-47c1-a2fe-6b1e94f37f01 + id: 3a8adcf5-1de5-44df-86dc-5d4180aed887 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: better_B_A + reference: '' + 455f42d8-caa8-4890-9545-607d3ef32e90: !Template + answer_choices: Yes ||| No + id: 455f42d8-caa8-4890-9545-607d3ef32e90 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes + reference: '' + 7b225c23-9765-44e5-b051-be845dddeeed: !Template + answer_choices: Yes ||| No + id: 7b225c23-9765-44e5-b051-be845dddeeed + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 95b23722-4db7-4283-8937-1f2ba5e67f25: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 95b23722-4db7-4283-8937-1f2ba5e67f25 + jinja: '" ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + 9eee0140-1bb2-40cc-a363-1f24dd0de19c: !Template + answer_choices: 1 ||| 2 + id: 9eee0140-1bb2-40cc-a363-1f24dd0de19c + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} {% endif %} @@ -38,14 +119,25 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: grammatical_which_one_1_2 reference: '' - 17dc5b32-9315-430d-9958-9da439352732: !Template + a8a33207-5db2-4f02-bce1-347d6da4c77e: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: a8a33207-5db2-4f02-bce1-347d6da4c77e + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: true_null_prompt + reference: '' + aee17b77-9972-42a5-8e2b-d9893bfcdfe8: !Template answer_choices: 1 ||| 2 - id: 17dc5b32-9315-430d-9958-9da439352732 + id: aee17b77-9972-42a5-8e2b-d9893bfcdfe8 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -77,14 +169,30 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: grammatical_between_1_2 + reference: '' + ba0af8f4-f5c5-4859-bab5-e43463e47c85: !Template + answer_choices: Yes ||| No + id: ba0af8f4-f5c5-4859-bab5-e43463e47c85 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no reference: '' - 399524ee-c0c0-436d-994b-4cda3d7586c5: !Template + db6a7631-9bfb-4c51-ad55-39bea946c4a6: !Template answer_choices: A ||| B - id: 399524ee-c0c0-436d-994b-4cda3d7586c5 + id: db6a7631-9bfb-4c51-ad55-39bea946c4a6 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -116,11 +224,11 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: grammatical_between_A_B reference: '' - 44ffe4cd-2787-4482-bc3f-a134a0d7db6c: !Template + ed86f3fd-fcec-4f33-8298-d0eeeb82051c: !Template answer_choices: A ||| B - id: 44ffe4cd-2787-4482-bc3f-a134a0d7db6c + id: ed86f3fd-fcec-4f33-8298-d0eeeb82051c jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -139,7 +247,7 @@ templates: {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order + Which one is a better sentence of English, A or B? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -154,11 +262,11 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: better_A_B reference: '' - 4b89b698-6c79-4eb0-9d19-2cac69b4cf3f: !Template + fa45e88d-fa9b-4165-803c-03aa61295cce: !Template answer_choices: Yes ||| No - id: 4b89b698-6c79-4eb0-9d19-2cac69b4cf3f + id: fa45e88d-fa9b-4165-803c-03aa61295cce jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -172,72 +280,24 @@ templates: original_task: false name: single_sentence_bad_no_yes reference: '' - 5fb3ddb1-515b-45b7-b2bf-b6c3cee673e0: !Template - answer_choices: Yes ||| No - id: 5fb3ddb1-515b-45b7-b2bf-b6c3cee673e0 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes - reference: '' - ddea312a-bbf9-49d8-808a-834462fb44b9: !Template - answer_choices: Yes ||| No - id: ddea312a-bbf9-49d8-808a-834462fb44b9 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no - reference: '' - e155bb05-1ef7-4f23-a1f3-3241fb177b0e: !Template - answer_choices: Yes ||| No - id: e155bb05-1ef7-4f23-a1f3-3241fb177b0e - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no - reference: '' - f1b75254-853d-4c0b-9de2-f6363138639c: !Template - answer_choices: 1 ||| 2 - id: f1b75254-853d-4c0b-9de2-f6363138639c + fcb7c59f-0e54-40c3-a444-ddacf0d67c1f: !Template + answer_choices: A ||| B + id: fcb7c59f-0e54-40c3-a444-ddacf0d67c1f jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -257,43 +317,5 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 - reference: '' - fa56226a-16f3-483c-b8bb-864d884c67aa: !Template - answer_choices: A ||| B - id: fa56226a-16f3-483c-b8bb-864d884c67aa - jinja: '{% set shuffled_order = [0, 1] | random %} - - Read the following two sentences. - - {% if shuffled_order == 0 %} - - Sentence A: {{ sentence_good }} - - Sentence B: {{ sentence_bad }} - - {% else %} - - Sentence A: {{ sentence_bad }} - - Sentence B: {{ sentence_good }} - - {% endif %} - - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} - - {% endif %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: better_A_B + name: grammatical_which_one_A_B reference: '' diff --git a/promptsource/templates/blimp/left_branch_island_simple_question/templates.yaml b/promptsource/templates/blimp/left_branch_island_simple_question/templates.yaml index 93a9321ca..7964991ec 100644 --- a/promptsource/templates/blimp/left_branch_island_simple_question/templates.yaml +++ b/promptsource/templates/blimp/left_branch_island_simple_question/templates.yaml @@ -1,24 +1,62 @@ dataset: blimp subset: left_branch_island_simple_question templates: - 27a3f691-e5b1-49a3-8a2d-a5f8c63b9a21: !Template + 08143a2e-5071-49c6-800b-dc41b3053d6c: !Template answer_choices: A ||| B - id: 27a3f691-e5b1-49a3-8a2d-a5f8c63b9a21 + id: 08143a2e-5071-49c6-800b-dc41b3053d6c jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: better_B_A + reference: '' + 1de92555-bf4a-40a6-8717-9f2fc3aaf5e0: !Template + answer_choices: 1 ||| 2 + id: 1de92555-bf4a-40a6-8717-9f2fc3aaf5e0 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} {% endif %} @@ -38,11 +76,11 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: grammatical_which_one_1_2 reference: '' - 30ce7c9b-cbb3-4f4c-9298-086cbd601257: !Template + 1f1d88ad-d90f-42c7-b2b4-bd36b2fecae7: !Template answer_choices: A ||| B - id: 30ce7c9b-cbb3-4f4c-9298-086cbd601257 + id: 1f1d88ad-d90f-42c7-b2b4-bd36b2fecae7 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -61,7 +99,7 @@ templates: {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order + Which one is a better sentence of English, A or B? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -76,27 +114,11 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A - reference: '' - 33c54e67-6fad-4c75-ab25-0e4fa2737e56: !Template - answer_choices: Yes ||| No - id: 33c54e67-6fad-4c75-ab25-0e4fa2737e56 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes + name: better_A_B reference: '' - 523c563f-7db2-4390-bd69-7d8f225bdd77: !Template + 1fd3ab83-7db4-4504-b26e-3450fccf8041: !Template answer_choices: Yes ||| No - id: 523c563f-7db2-4390-bd69-7d8f225bdd77 + id: 1fd3ab83-7db4-4504-b26e-3450fccf8041 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -110,12 +132,12 @@ templates: original_task: false name: single_sentence_good_no_yes reference: '' - 6d29c6bb-b345-4e0e-9b9c-9a7b05b5c2b3: !Template + 2a0675ad-6dc1-47ff-8456-09b6061973f1: !Template answer_choices: A ||| B - id: 6d29c6bb-b345-4e0e-9b9c-9a7b05b5c2b3 + id: 2a0675ad-6dc1-47ff-8456-09b6061973f1 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -147,30 +169,14 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B - reference: '' - 7fafffeb-e768-4abe-97e4-586e8043d477: !Template - answer_choices: Yes ||| No - id: 7fafffeb-e768-4abe-97e4-586e8043d477 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no + name: grammatical_between_A_B reference: '' - 92c91987-ab83-45aa-bfb6-5ac7382e253c: !Template + 480140c6-d094-4111-9740-4cf92b62cbdc: !Template answer_choices: 1 ||| 2 - id: 92c91987-ab83-45aa-bfb6-5ac7382e253c + id: 480140c6-d094-4111-9740-4cf92b62cbdc jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -202,15 +208,15 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: grammatical_between_1_2 reference: '' - ad77d7b0-6cb3-4fe8-af1d-190707cb2ad2: !Template + 96a9b402-ab24-411c-9f7b-5b9df0ceb6c5: !Template answer_choices: Yes ||| No - id: ad77d7b0-6cb3-4fe8-af1d-190707cb2ad2 + id: 96a9b402-ab24-411c-9f7b-5b9df0ceb6c5 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata @@ -218,64 +224,80 @@ templates: metrics: - Accuracy original_task: false - name: single_sentence_bad_yes_no + name: single_sentence_bad_no_yes reference: '' - f6f176e5-b975-431c-a0fe-fca1ba07099f: !Template - answer_choices: A ||| B - id: f6f176e5-b975-431c-a0fe-fca1ba07099f - jinja: '{% set shuffled_order = [0, 1] | random %} - - Read the following two sentences. - - {% if shuffled_order == 0 %} - - Sentence A: {{ sentence_good }} - - Sentence B: {{ sentence_bad }} - - {% else %} - - Sentence A: {{ sentence_bad }} - - Sentence B: {{ sentence_good }} - - {% endif %} - - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} + 99a676b2-2e8a-4e11-918f-09b1fe76253a: !Template + answer_choices: Yes ||| No + id: 99a676b2-2e8a-4e11-918f-09b1fe76253a + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - {% endif %}' + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + ad08cfc7-2ffa-48a9-b86b-fbbb5da87d90: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: ad08cfc7-2ffa-48a9-b86b-fbbb5da87d90 + jinja: '" ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy original_task: true - name: better_A_B + name: null_prompt reference: '' - f945855c-7f51-4542-920f-855c44b11f96: !Template - answer_choices: 1 ||| 2 - id: f945855c-7f51-4542-920f-855c44b11f96 + cc5a36ad-b6dd-4e18-8558-87d074b26e0d: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: cc5a36ad-b6dd-4e18-8558-87d074b26e0d + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: true_null_prompt + reference: '' + d6c43902-2bce-43c0-8129-936880506031: !Template + answer_choices: Yes ||| No + id: d6c43902-2bce-43c0-8129-936880506031 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + f90dbd74-7530-436f-b0f8-8f2b9658e858: !Template + answer_choices: A ||| B + id: f90dbd74-7530-436f-b0f8-8f2b9658e858 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -295,5 +317,5 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: grammatical_which_one_A_B reference: '' diff --git a/promptsource/templates/blimp/matrix_question_npi_licensor_present/templates.yaml b/promptsource/templates/blimp/matrix_question_npi_licensor_present/templates.yaml index 174cd912d..8fdb7c263 100644 --- a/promptsource/templates/blimp/matrix_question_npi_licensor_present/templates.yaml +++ b/promptsource/templates/blimp/matrix_question_npi_licensor_present/templates.yaml @@ -1,24 +1,35 @@ dataset: blimp subset: matrix_question_npi_licensor_present templates: - 07b48aa1-d407-462a-b577-25aabf040692: !Template - answer_choices: 1 ||| 2 - id: 07b48aa1-d407-462a-b577-25aabf040692 + 0c3e2da6-df0c-4bca-8ed0-e95cfa0d8f9d: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 0c3e2da6-df0c-4bca-8ed0-e95cfa0d8f9d + jinja: '" ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + 2cfa7c09-25ff-4e66-a07a-886d868a3ed3: !Template + answer_choices: A ||| B + id: 2cfa7c09-25ff-4e66-a07a-886d868a3ed3 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -38,31 +49,32 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: grammatical_between_A_B reference: '' - 406d268d-3468-4965-a875-443b306b5615: !Template - answer_choices: A ||| B - id: 406d268d-3468-4965-a875-443b306b5615 + 324780ec-3fa7-4b20-ba5d-a2cf8dbc2e0d: !Template + answer_choices: 1 ||| 2 + id: 324780ec-3fa7-4b20-ba5d-a2cf8dbc2e0d jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -76,26 +88,26 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: grammatical_between_1_2 reference: '' - 43e2da7e-7a7b-4021-84ad-814767601f3f: !Template - answer_choices: A ||| B - id: 43e2da7e-7a7b-4021-84ad-814767601f3f + 63c864fc-0e20-4548-99bf-498d6f55e491: !Template + answer_choices: 1 ||| 2 + id: 63c864fc-0e20-4548-99bf-498d6f55e491 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -115,11 +127,11 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: grammatical_which_one_1_2 reference: '' - 5e9a2cdb-70a4-4923-ad49-b2cd13c940c2: !Template + 97d1289e-9ed4-41b1-9d67-4c9d24842fc4: !Template answer_choices: Yes ||| No - id: 5e9a2cdb-70a4-4923-ad49-b2cd13c940c2 + id: 97d1289e-9ed4-41b1-9d67-4c9d24842fc4 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -133,13 +145,13 @@ templates: original_task: false name: single_sentence_bad_no_yes reference: '' - 6a3849a9-12db-42f6-8de6-79bc6cf648b5: !Template + 9a4e55cc-1252-4200-8510-0263b2343e44: !Template answer_choices: Yes ||| No - id: 6a3849a9-12db-42f6-8de6-79bc6cf648b5 + id: 9a4e55cc-1252-4200-8510-0263b2343e44 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -147,32 +159,31 @@ templates: metrics: - Accuracy original_task: false - name: single_sentence_good_yes_no + name: single_sentence_good_no_yes reference: '' - 7d510aad-649c-4b6b-a18b-537a336fe4e1: !Template + a8a277d4-1aa4-41dd-b243-085bee5595aa: !Template answer_choices: A ||| B - id: 7d510aad-649c-4b6b-a18b-537a336fe4e1 + id: a8a277d4-1aa4-41dd-b243-085bee5595aa jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -186,64 +197,31 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B - reference: '' - 83c9ae3f-92a4-4391-b7e1-a2cf107e7a01: !Template - answer_choices: Yes ||| No - id: 83c9ae3f-92a4-4391-b7e1-a2cf107e7a01 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no - reference: '' - 8fdbc6db-e6a7-4f7a-935e-64fd13f9f23d: !Template - answer_choices: Yes ||| No - id: 8fdbc6db-e6a7-4f7a-935e-64fd13f9f23d - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes + name: better_B_A reference: '' - cf8ca906-3501-4036-8360-1c1e0f5da1a0: !Template - answer_choices: 1 ||| 2 - id: cf8ca906-3501-4036-8360-1c1e0f5da1a0 + b7107e84-8173-40d5-aa22-8a477461381b: !Template + answer_choices: A ||| B + id: b7107e84-8173-40d5-aa22-8a477461381b jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -257,31 +235,48 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: better_A_B reference: '' - db3ba96d-9b12-48eb-ba18-61b8f57a94be: !Template + ba7709d8-1f30-4c08-b059-c6ec0ebb68c8: !Template + answer_choices: Yes ||| No + id: ba7709d8-1f30-4c08-b059-c6ec0ebb68c8 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + bb48ee5e-9c07-4cc7-a671-2984e1d4ef90: !Template answer_choices: A ||| B - id: db3ba96d-9b12-48eb-ba18-61b8f57a94be + id: bb48ee5e-9c07-4cc7-a671-2984e1d4ef90 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -295,5 +290,32 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: grammatical_which_one_A_B + reference: '' + f712b9af-fe4b-4927-a39b-c35d0ad2482d: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: f712b9af-fe4b-4927-a39b-c35d0ad2482d + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: true_null_prompt + reference: '' + f9f2602a-6225-418e-8a51-9cbdec53daa0: !Template + answer_choices: Yes ||| No + id: f9f2602a-6225-418e-8a51-9cbdec53daa0 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no reference: '' diff --git a/promptsource/templates/blimp/npi_present_1/templates.yaml b/promptsource/templates/blimp/npi_present_1/templates.yaml index a0bb15987..9e88762db 100644 --- a/promptsource/templates/blimp/npi_present_1/templates.yaml +++ b/promptsource/templates/blimp/npi_present_1/templates.yaml @@ -1,13 +1,13 @@ dataset: blimp subset: npi_present_1 templates: - 0ec6e8a2-6c7e-4029-899c-977d898dc720: !Template + 0f05dfa3-8987-4158-bfbb-6e722cd4ac6b: !Template answer_choices: Yes ||| No - id: 0ec6e8a2-6c7e-4029-899c-977d898dc720 + id: 0f05dfa3-8987-4158-bfbb-6e722cd4ac6b jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata @@ -15,26 +15,48 @@ templates: metrics: - Accuracy original_task: false - name: single_sentence_bad_no_yes + name: single_sentence_bad_yes_no reference: '' - 295ddef9-8094-4241-9692-079048aa4cf8: !Template - answer_choices: 1 ||| 2 - id: 295ddef9-8094-4241-9692-079048aa4cf8 + 3430b62d-4b7d-4edf-bba1-8ee873c6d250: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 3430b62d-4b7d-4edf-bba1-8ee873c6d250 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: true_null_prompt + reference: '' + 3ab2d04c-7ccb-41ba-93d7-0a6b8d918248: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 3ab2d04c-7ccb-41ba-93d7-0a6b8d918248 + jinja: '" ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + 441d513b-b700-4949-bd4e-ed6d52867266: !Template + answer_choices: A ||| B + id: 441d513b-b700-4949-bd4e-ed6d52867266 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -54,27 +76,11 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 - reference: '' - 3061fc20-3a06-4463-81dd-19eba967136e: !Template - answer_choices: Yes ||| No - id: 3061fc20-3a06-4463-81dd-19eba967136e - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no + name: grammatical_between_A_B reference: '' - 3625ff42-2148-45b9-9193-dfd98bb285cc: !Template + 53522fa9-e908-40af-bda2-fbb0af141047: !Template answer_choices: 1 ||| 2 - id: 3625ff42-2148-45b9-9193-dfd98bb285cc + id: 53522fa9-e908-40af-bda2-fbb0af141047 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -111,9 +117,9 @@ templates: original_task: true name: grammatical_between_1_2 reference: '' - 3756c6cc-6f26-4f6a-b74c-4e5699ff39a2: !Template + 707984c7-d933-4a85-b5d1-e011321b41e9: !Template answer_choices: A ||| B - id: 3756c6cc-6f26-4f6a-b74c-4e5699ff39a2 + id: 707984c7-d933-4a85-b5d1-e011321b41e9 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -149,30 +155,61 @@ templates: original_task: true name: better_A_B reference: '' - 55004e01-cbff-451d-b023-17f59cb9bbb2: !Template + 83c92e79-ee02-4bd6-8583-3dcdcf466e8a: !Template + answer_choices: Yes ||| No + id: 83c92e79-ee02-4bd6-8583-3dcdcf466e8a + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes + reference: '' + 94212f12-3828-4123-aa16-bfe82bf067e1: !Template + answer_choices: Yes ||| No + id: 94212f12-3828-4123-aa16-bfe82bf067e1 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes + reference: '' + 9d489f52-223c-4ba5-9e48-7024245eefd6: !Template answer_choices: A ||| B - id: 55004e01-cbff-451d-b023-17f59cb9bbb2 + id: 9d489f52-223c-4ba5-9e48-7024245eefd6 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -186,27 +223,11 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B - reference: '' - 5a855dac-928d-49bd-8e7e-843a27bbda2e: !Template - answer_choices: Yes ||| No - id: 5a855dac-928d-49bd-8e7e-843a27bbda2e - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes + name: better_B_A reference: '' - 6326ea5b-a19c-468a-9abd-fb7f69ccb596: !Template + acb76db2-c9f5-49fb-87b6-4f12a05e22c6: !Template answer_choices: Yes ||| No - id: 6326ea5b-a19c-468a-9abd-fb7f69ccb596 + id: acb76db2-c9f5-49fb-87b6-4f12a05e22c6 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -220,24 +241,24 @@ templates: original_task: false name: single_sentence_good_yes_no reference: '' - 90f4ce45-a860-4124-aa2f-3bfd35acbd75: !Template - answer_choices: A ||| B - id: 90f4ce45-a860-4124-aa2f-3bfd35acbd75 + bb82ebdd-bf2d-45f4-ba14-6efa4c5495d5: !Template + answer_choices: 1 ||| 2 + id: bb82ebdd-bf2d-45f4-ba14-6efa4c5495d5 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -257,31 +278,32 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: grammatical_which_one_1_2 reference: '' - c0024b90-613a-4b4b-88bf-beeb05400208: !Template + f679d29f-ed40-4bda-a640-c8fc125cd315: !Template answer_choices: A ||| B - id: c0024b90-613a-4b4b-88bf-beeb05400208 + id: f679d29f-ed40-4bda-a640-c8fc125cd315 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -295,5 +317,5 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: grammatical_which_one_A_B reference: '' diff --git a/promptsource/templates/blimp/npi_present_2/templates.yaml b/promptsource/templates/blimp/npi_present_2/templates.yaml index 2432b0c7d..2a50689f7 100644 --- a/promptsource/templates/blimp/npi_present_2/templates.yaml +++ b/promptsource/templates/blimp/npi_present_2/templates.yaml @@ -1,40 +1,24 @@ dataset: blimp subset: npi_present_2 templates: - 0b89e24c-7d21-4955-ae69-f3c472a026b8: !Template - answer_choices: Yes ||| No - id: 0b89e24c-7d21-4955-ae69-f3c472a026b8 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes - reference: '' - 3993b6bc-0ac2-46c3-977e-fb57f9817386: !Template - answer_choices: 1 ||| 2 - id: 3993b6bc-0ac2-46c3-977e-fb57f9817386 + 189f7c26-e8ee-40df-9e2b-cd2438d2b8ca: !Template + answer_choices: A ||| B + id: 189f7c26-e8ee-40df-9e2b-cd2438d2b8ca jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -54,26 +38,26 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: grammatical_which_one_A_B reference: '' - 40251c2f-cd5b-4325-a7a0-37ad7a518376: !Template - answer_choices: A ||| B - id: 40251c2f-cd5b-4325-a7a0-37ad7a518376 + 18d8882d-7f23-420f-a321-d8b3a039ce86: !Template + answer_choices: 1 ||| 2 + id: 18d8882d-7f23-420f-a321-d8b3a039ce86 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -93,32 +77,47 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: grammatical_which_one_1_2 reference: '' - 62439fab-8be3-47d9-968c-23af62dfae28: !Template + 1f9b7c05-c643-4434-8315-624b01b7af9f: !Template + answer_choices: Yes ||| No + id: 1f9b7c05-c643-4434-8315-624b01b7af9f + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 2c6c8dcc-6c99-45e2-a393-db259665e794: !Template answer_choices: A ||| B - id: 62439fab-8be3-47d9-968c-23af62dfae28 + id: 2c6c8dcc-6c99-45e2-a393-db259665e794 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -132,11 +131,27 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: better_B_A + reference: '' + 513f5f9f-10b5-4704-ac5e-781e30bb3e22: !Template + answer_choices: Yes ||| No + id: 513f5f9f-10b5-4704-ac5e-781e30bb3e22 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes reference: '' - 913c63c8-c47a-4fca-b2b9-24536c2b2e55: !Template + 5b795bc2-82d5-442c-bee0-e542b1d647ae: !Template answer_choices: A ||| B - id: 913c63c8-c47a-4fca-b2b9-24536c2b2e55 + id: 5b795bc2-82d5-442c-bee0-e542b1d647ae jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -172,45 +187,30 @@ templates: original_task: true name: better_A_B reference: '' - 9616903f-3cc0-4a4b-91dc-1a34a0e05e94: !Template - answer_choices: Yes ||| No - id: 9616903f-3cc0-4a4b-91dc-1a34a0e05e94 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no - reference: '' - aa523410-dc51-4aac-bfa9-e98263a39f31: !Template - answer_choices: A ||| B - id: aa523410-dc51-4aac-bfa9-e98263a39f31 + 5d3d3930-eb49-4f0d-9e8c-6fa3ac8bc309: !Template + answer_choices: 1 ||| 2 + id: 5d3d3930-eb49-4f0d-9e8c-6fa3ac8bc309 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -224,11 +224,22 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: grammatical_between_1_2 reference: '' - accced8c-ab98-4944-b2b1-348933bdcf05: !Template + 89120a35-57b5-4654-b35d-3fa817d64b18: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 89120a35-57b5-4654-b35d-3fa817d64b18 + jinja: '" ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + b084e594-0a98-4745-add5-291afcecff39: !Template answer_choices: Yes ||| No - id: accced8c-ab98-4944-b2b1-348933bdcf05 + id: b084e594-0a98-4745-add5-291afcecff39 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -242,40 +253,51 @@ templates: original_task: false name: single_sentence_good_yes_no reference: '' - f33d2c45-e5d6-450c-8252-e57c8638d921: !Template + db6c97b6-2518-40d1-9e73-809b2a259cf4: !Template answer_choices: Yes ||| No - id: f33d2c45-e5d6-450c-8252-e57c8638d921 + id: db6c97b6-2518-40d1-9e73-809b2a259cf4 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_good_no_yes + name: single_sentence_bad_no_yes reference: '' - fcb15bb8-18ce-4a4d-80df-5c7d61d8a1e2: !Template - answer_choices: 1 ||| 2 - id: fcb15bb8-18ce-4a4d-80df-5c7d61d8a1e2 + dccf9f87-07ed-4efb-a54e-5e3883b802b6: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: dccf9f87-07ed-4efb-a54e-5e3883b802b6 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: true_null_prompt + reference: '' + e3854983-90a4-4b88-8628-7cd3652e638d: !Template + answer_choices: A ||| B + id: e3854983-90a4-4b88-8628-7cd3652e638d jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -295,5 +317,5 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: grammatical_between_A_B reference: '' diff --git a/promptsource/templates/blimp/only_npi_licensor_present/templates.yaml b/promptsource/templates/blimp/only_npi_licensor_present/templates.yaml index 6611931f5..43ed33289 100644 --- a/promptsource/templates/blimp/only_npi_licensor_present/templates.yaml +++ b/promptsource/templates/blimp/only_npi_licensor_present/templates.yaml @@ -1,9 +1,20 @@ dataset: blimp subset: only_npi_licensor_present templates: - 1b9ca899-4b67-4de0-8c43-4539a71fa4b8: !Template + 2b35f684-b468-46db-977e-d2c0bfd3db00: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 2b35f684-b468-46db-977e-d2c0bfd3db00 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: true_null_prompt + reference: '' + 2e45cdb6-9f51-45ff-9d80-5537b69d7ca6: !Template answer_choices: A ||| B - id: 1b9ca899-4b67-4de0-8c43-4539a71fa4b8 + id: 2e45cdb6-9f51-45ff-9d80-5537b69d7ca6 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -39,56 +50,24 @@ templates: original_task: true name: better_B_A reference: '' - 37d70401-ea98-4863-99ac-ab652ce94678: !Template - answer_choices: Yes ||| No - id: 37d70401-ea98-4863-99ac-ab652ce94678 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes - reference: '' - 746cf635-d43d-453c-bfe6-c397cd971e93: !Template - answer_choices: Yes ||| No - id: 746cf635-d43d-453c-bfe6-c397cd971e93 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no - reference: '' - 95ea0068-8f46-4d77-a9a0-43e659ba5453: !Template - answer_choices: 1 ||| 2 - id: 95ea0068-8f46-4d77-a9a0-43e659ba5453 + 58893642-b4d2-498b-854d-475b05c18fd6: !Template + answer_choices: A ||| B + id: 58893642-b4d2-498b-854d-475b05c18fd6 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -108,42 +87,26 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 - reference: '' - abe1b796-d2d8-49b6-a86a-cb5e7325d699: !Template - answer_choices: Yes ||| No - id: abe1b796-d2d8-49b6-a86a-cb5e7325d699 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes + name: grammatical_which_one_A_B reference: '' - b6fef585-cb6c-4fd7-9b9d-4791d1a94951: !Template - answer_choices: 1 ||| 2 - id: b6fef585-cb6c-4fd7-9b9d-4791d1a94951 + 5ce28040-abd8-4e7f-a3a7-628550ed4604: !Template + answer_choices: A ||| B + id: 5ce28040-abd8-4e7f-a3a7-628550ed4604 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -163,26 +126,26 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: grammatical_between_A_B reference: '' - ca2daa84-9dd4-4627-ad69-3b2df3cedd38: !Template - answer_choices: A ||| B - id: ca2daa84-9dd4-4627-ad69-3b2df3cedd38 + 6d77f04a-3602-4aad-9842-796008030b91: !Template + answer_choices: 1 ||| 2 + id: 6d77f04a-3602-4aad-9842-796008030b91 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -202,11 +165,11 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: grammatical_which_one_1_2 reference: '' - cfceb31c-4da1-4317-88d5-65be913ee254: !Template + 8215ed1e-f8cb-4b4e-af6f-3f618204ff0d: !Template answer_choices: Yes ||| No - id: cfceb31c-4da1-4317-88d5-65be913ee254 + id: 8215ed1e-f8cb-4b4e-af6f-3f618204ff0d jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -220,30 +183,56 @@ templates: original_task: false name: single_sentence_bad_yes_no reference: '' - ec13eac1-3af6-4558-a70d-51a228d776b5: !Template + 8a622aa9-0dd1-44ac-aade-269b9e0848e0: !Template + answer_choices: Yes ||| No + id: 8a622aa9-0dd1-44ac-aade-269b9e0848e0 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes + reference: '' + 8e69a142-f645-4b24-8838-82b14a975335: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 8e69a142-f645-4b24-8838-82b14a975335 + jinja: '" ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + ab945f21-42f5-42d7-971b-5bc7f407ed87: !Template answer_choices: A ||| B - id: ec13eac1-3af6-4558-a70d-51a228d776b5 + id: ab945f21-42f5-42d7-971b-5bc7f407ed87 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -257,31 +246,64 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: better_A_B reference: '' - fc585a04-0b35-42cd-86bd-34cbdd074fd5: !Template - answer_choices: A ||| B - id: fc585a04-0b35-42cd-86bd-34cbdd074fd5 + bba80804-0dd8-4448-9fb9-96d25dcd7bcf: !Template + answer_choices: Yes ||| No + id: bba80804-0dd8-4448-9fb9-96d25dcd7bcf + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + ced631e8-585a-4baa-90c5-b06554a89e6a: !Template + answer_choices: Yes ||| No + id: ced631e8-585a-4baa-90c5-b06554a89e6a + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes + reference: '' + e9de0279-7efc-40ea-bf52-f58a4d0eb079: !Template + answer_choices: 1 ||| 2 + id: e9de0279-7efc-40ea-bf52-f58a4d0eb079 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -295,5 +317,5 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: grammatical_between_1_2 reference: '' diff --git a/promptsource/templates/blimp/only_npi_scope/templates.yaml b/promptsource/templates/blimp/only_npi_scope/templates.yaml index f7851ba01..1945eb750 100644 --- a/promptsource/templates/blimp/only_npi_scope/templates.yaml +++ b/promptsource/templates/blimp/only_npi_scope/templates.yaml @@ -1,84 +1,57 @@ dataset: blimp subset: only_npi_scope templates: - 3482c891-a0ba-4554-b669-64be4a25975d: !Template - answer_choices: 1 ||| 2 - id: 3482c891-a0ba-4554-b669-64be4a25975d - jinja: '{% set shuffled_order = [0, 1] | random %} - - Which one of the following sentences is grammatical? Please answer 1 or 2. - - {% if shuffled_order == 0 %} - - 1: {{ sentence_good }} - - 2: {{ sentence_bad }} - - {% else %} - - 1: {{ sentence_bad }} - - 2: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} - - {% endif %}' + 09c6c213-08ff-4e54-8651-8506dd1c1c23: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 09c6c213-08ff-4e54-8651-8506dd1c1c23 + jinja: ' ||| {{ sentence_good }}' metadata: !TemplateMetadata - choices_in_prompt: true + choices_in_prompt: false metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: true_null_prompt reference: '' - 3ca02c3f-b858-4e4c-8260-21bf2c1a2b34: !Template + 0ac1af63-efc8-4320-ad84-19aecaf5e18b: !Template answer_choices: Yes ||| No - id: 3ca02c3f-b858-4e4c-8260-21bf2c1a2b34 + id: 0ac1af63-efc8-4320-ad84-19aecaf5e18b jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_good_yes_no + name: single_sentence_bad_no_yes reference: '' - 5e9be923-8cf8-41e9-b384-4ab54d473cd4: !Template + 25eae00b-576f-4e77-9869-3c1108d83e84: !Template answer_choices: A ||| B - id: 5e9be923-8cf8-41e9-b384-4ab54d473cd4 + id: 25eae00b-576f-4e77-9869-3c1108d83e84 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -92,11 +65,27 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: grammatical_between_A_B reference: '' - af1d5046-01a1-4036-879d-cbf1952c3b03: !Template + 26402983-4cc9-4f60-8876-6abb2e668af1: !Template + answer_choices: Yes ||| No + id: 26402983-4cc9-4f60-8876-6abb2e668af1 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes + reference: '' + 2ec5cad4-5f63-4e8b-89f4-89fa23d77051: !Template answer_choices: 1 ||| 2 - id: af1d5046-01a1-4036-879d-cbf1952c3b03 + id: 2ec5cad4-5f63-4e8b-89f4-89fa23d77051 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -133,28 +122,28 @@ templates: original_task: true name: grammatical_between_1_2 reference: '' - c80b4a8c-c786-4c45-886f-37c7a26a67b7: !Template + 5c438c94-a594-40f9-8e37-0566363bde37: !Template answer_choices: Yes ||| No - id: c80b4a8c-c786-4c45-886f-37c7a26a67b7 + id: 5c438c94-a594-40f9-8e37-0566363bde37 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_good_no_yes + name: single_sentence_bad_yes_no reference: '' - de665624-ec21-4256-b745-5c0380a10c31: !Template + 7b41f9cd-24a0-4436-b4eb-36a433ba72a1: !Template answer_choices: A ||| B - id: de665624-ec21-4256-b745-5c0380a10c31 + id: 7b41f9cd-24a0-4436-b4eb-36a433ba72a1 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -186,11 +175,11 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: grammatical_which_one_A_B reference: '' - de7f9e83-effa-4aba-a3c5-695678c57b37: !Template + 85afff0b-e9d9-497f-b7d6-7ffa9a719a26: !Template answer_choices: A ||| B - id: de7f9e83-effa-4aba-a3c5-695678c57b37 + id: 85afff0b-e9d9-497f-b7d6-7ffa9a719a26 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -209,7 +198,7 @@ templates: {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order + Which one is a better sentence of English, A or B? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -224,58 +213,91 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: better_A_B reference: '' - e876bb66-ec5b-4fa8-be20-0ea7ff7ff92c: !Template - answer_choices: Yes ||| No - id: e876bb66-ec5b-4fa8-be20-0ea7ff7ff92c - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + af88d64d-85b6-4cc5-ba7b-4a6dc5d897a8: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: af88d64d-85b6-4cc5-ba7b-4a6dc5d897a8 + jinja: '" ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + c6532a04-941c-4a74-8939-910a95b7338f: !Template + answer_choices: A ||| B + id: c6532a04-941c-4a74-8939-910a95b7338f + jinja: '{% set shuffled_order = [0, 1] | random %} - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: false - name: single_sentence_bad_no_yes + original_task: true + name: better_B_A reference: '' - f35d59da-a22a-444f-8762-c878586a054f: !Template + c8d407aa-e94d-4ac1-a7f5-8607de064195: !Template answer_choices: Yes ||| No - id: f35d59da-a22a-444f-8762-c878586a054f + id: c8d407aa-e94d-4ac1-a7f5-8607de064195 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_bad_yes_no + name: single_sentence_good_yes_no reference: '' - f98213c5-0013-4edc-a0b3-7e0ac1b3738c: !Template - answer_choices: A ||| B - id: f98213c5-0013-4edc-a0b3-7e0ac1b3738c + dfa910b3-6aa2-4534-afc6-6072fd30de20: !Template + answer_choices: 1 ||| 2 + id: dfa910b3-6aa2-4534-afc6-6072fd30de20 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -295,5 +317,5 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: grammatical_which_one_1_2 reference: '' diff --git a/promptsource/templates/blimp/passive_1/templates.yaml b/promptsource/templates/blimp/passive_1/templates.yaml index 9c5ce8a11..1c9110f38 100644 --- a/promptsource/templates/blimp/passive_1/templates.yaml +++ b/promptsource/templates/blimp/passive_1/templates.yaml @@ -1,24 +1,24 @@ dataset: blimp subset: passive_1 templates: - 0eafeb00-4c83-423a-8b6e-9cf3f50f6514: !Template - answer_choices: A ||| B - id: 0eafeb00-4c83-423a-8b6e-9cf3f50f6514 + 25bd86e2-da2e-45b5-a7eb-ffbb4cb4e3c6: !Template + answer_choices: 1 ||| 2 + id: 25bd86e2-da2e-45b5-a7eb-ffbb4cb4e3c6 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -38,48 +38,31 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B - reference: '' - 289e788b-2c6a-4e27-bed5-787b3a301783: !Template - answer_choices: Yes ||| No - id: 289e788b-2c6a-4e27-bed5-787b3a301783 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes + name: grammatical_between_1_2 reference: '' - 63202675-475d-4384-8309-6a1cec1a4436: !Template - answer_choices: 1 ||| 2 - id: 63202675-475d-4384-8309-6a1cec1a4436 + 4cd96a94-68e3-4c21-88cd-3f5cab3f9b22: !Template + answer_choices: A ||| B + id: 4cd96a94-68e3-4c21-88cd-3f5cab3f9b22 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -93,11 +76,70 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: better_A_B + reference: '' + 63f89191-36ab-4e9e-ba46-ff2051b59879: !Template + answer_choices: Yes ||| No + id: 63f89191-36ab-4e9e-ba46-ff2051b59879 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 7cf041bf-4063-46b0-88f1-dc06bbe2fe30: !Template + answer_choices: Yes ||| No + id: 7cf041bf-4063-46b0-88f1-dc06bbe2fe30 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 7da525bb-6e3c-4ae0-82d3-9080c7684999: !Template + answer_choices: Yes ||| No + id: 7da525bb-6e3c-4ae0-82d3-9080c7684999 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes + reference: '' + 883d09a8-64b7-4f97-853a-8aee97cf9d6d: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 883d09a8-64b7-4f97-853a-8aee97cf9d6d + jinja: '" ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt reference: '' - 722af26b-dd76-48e5-a610-dd59fb06a22e: !Template + 9c6000cf-cab5-498a-a978-c34961b192a1: !Template answer_choices: A ||| B - id: 722af26b-dd76-48e5-a610-dd59fb06a22e + id: 9c6000cf-cab5-498a-a978-c34961b192a1 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -116,7 +158,7 @@ templates: {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order + Which one is a better sentence of English, B or A? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -131,14 +173,25 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: better_B_A reference: '' - 7809a036-4a53-48c7-9887-03e2366933b7: !Template + ae4277b8-f24a-4fc3-bed2-0096cf43e375: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: ae4277b8-f24a-4fc3-bed2-0096cf43e375 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: true_null_prompt + reference: '' + ba726648-94f2-43f0-9ded-513514570bc7: !Template answer_choices: 1 ||| 2 - id: 7809a036-4a53-48c7-9887-03e2366933b7 + id: ba726648-94f2-43f0-9ded-513514570bc7 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -170,31 +223,48 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: grammatical_which_one_1_2 reference: '' - b5d41404-3b11-4745-843e-527baeeb536e: !Template + c1de02ea-9280-4fef-902c-c9fc58c7e297: !Template + answer_choices: Yes ||| No + id: c1de02ea-9280-4fef-902c-c9fc58c7e297 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes + reference: '' + f2f4d471-a4ae-4b2a-8dbf-377720cb6e2e: !Template answer_choices: A ||| B - id: b5d41404-3b11-4745-843e-527baeeb536e + id: f2f4d471-a4ae-4b2a-8dbf-377720cb6e2e jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -208,11 +278,11 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: grammatical_between_A_B reference: '' - cfd42573-720e-4014-abba-3146aaa39ec1: !Template + f42993b2-eefd-4413-b786-01f98342a66f: !Template answer_choices: A ||| B - id: cfd42573-720e-4014-abba-3146aaa39ec1 + id: f42993b2-eefd-4413-b786-01f98342a66f jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer A or B. @@ -249,51 +319,3 @@ templates: original_task: true name: grammatical_which_one_A_B reference: '' - e1014909-5b42-41ff-9ed0-2e432975de51: !Template - answer_choices: Yes ||| No - id: e1014909-5b42-41ff-9ed0-2e432975de51 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes - reference: '' - ef7c1343-d328-4d10-a777-07cab171e674: !Template - answer_choices: Yes ||| No - id: ef7c1343-d328-4d10-a777-07cab171e674 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no - reference: '' - efc34c01-f855-49c8-830d-a60f3422ccc4: !Template - answer_choices: Yes ||| No - id: efc34c01-f855-49c8-830d-a60f3422ccc4 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no - reference: '' diff --git a/promptsource/templates/blimp/passive_2/templates.yaml b/promptsource/templates/blimp/passive_2/templates.yaml index 4785665a2..799d1a159 100644 --- a/promptsource/templates/blimp/passive_2/templates.yaml +++ b/promptsource/templates/blimp/passive_2/templates.yaml @@ -1,9 +1,20 @@ dataset: blimp subset: passive_2 templates: - 32f79d36-e1b0-4ce5-affc-8129311b6e22: !Template + 23f63696-2ef8-4b22-8498-8c4a281037f0: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 23f63696-2ef8-4b22-8498-8c4a281037f0 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: true_null_prompt + reference: '' + 3e13c0f4-d863-4c4c-9551-6d391d78999b: !Template answer_choices: A ||| B - id: 32f79d36-e1b0-4ce5-affc-8129311b6e22 + id: 3e13c0f4-d863-4c4c-9551-6d391d78999b jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer A or B. @@ -40,46 +51,29 @@ templates: original_task: true name: grammatical_between_A_B reference: '' - 361d1523-8535-450a-9162-db2d45344194: !Template - answer_choices: Yes ||| No - id: 361d1523-8535-450a-9162-db2d45344194 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes - reference: '' - 731b4c9b-2c37-4b1f-8a32-000f3ebdcac3: !Template - answer_choices: 1 ||| 2 - id: 731b4c9b-2c37-4b1f-8a32-000f3ebdcac3 + 4203849e-c999-428e-9dc8-45616735d6da: !Template + answer_choices: A ||| B + id: 4203849e-c999-428e-9dc8-45616735d6da jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -93,11 +87,11 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: better_B_A reference: '' - 7fd25e68-074f-498f-835a-44f1fac35a21: !Template + 4807d853-45bb-4baa-b4d3-21253fa651c6: !Template answer_choices: A ||| B - id: 7fd25e68-074f-498f-835a-44f1fac35a21 + id: 4807d853-45bb-4baa-b4d3-21253fa651c6 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer A or B. @@ -134,44 +128,28 @@ templates: original_task: true name: grammatical_which_one_A_B reference: '' - 96bf0a44-1f13-4fed-9fc0-3a817112cc67: !Template - answer_choices: Yes ||| No - id: 96bf0a44-1f13-4fed-9fc0-3a817112cc67 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes - reference: '' - 98b68892-3302-448a-8243-d2b178a3f522: !Template + 5f9f8395-3c0b-40e7-a860-2c7cbb776645: !Template answer_choices: Yes ||| No - id: 98b68892-3302-448a-8243-d2b178a3f522 + id: 5f9f8395-3c0b-40e7-a860-2c7cbb776645 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_bad_yes_no + name: single_sentence_good_no_yes reference: '' - be971081-1ba2-4a63-9c5a-9419e4555c93: !Template + 8029a698-b8d8-425c-b7e5-7fa02df6d5b0: !Template answer_choices: 1 ||| 2 - id: be971081-1ba2-4a63-9c5a-9419e4555c93 + id: 8029a698-b8d8-425c-b7e5-7fa02df6d5b0 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -203,31 +181,64 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: grammatical_between_1_2 reference: '' - c1fcde28-55da-41c7-8a76-58b0850aa762: !Template - answer_choices: A ||| B - id: c1fcde28-55da-41c7-8a76-58b0850aa762 + 82c53b24-7029-44bc-9215-db9a4b9e3240: !Template + answer_choices: Yes ||| No + id: 82c53b24-7029-44bc-9215-db9a4b9e3240 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 9fbfb530-eb17-437a-b8ef-6738e5c4246e: !Template + answer_choices: Yes ||| No + id: 9fbfb530-eb17-437a-b8ef-6738e5c4246e + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + b36882ca-2b6b-43e3-86b9-0acdb00d81a8: !Template + answer_choices: 1 ||| 2 + id: b36882ca-2b6b-43e3-86b9-0acdb00d81a8 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -241,11 +252,11 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: grammatical_which_one_1_2 reference: '' - d11adddf-ae9a-47e1-a7b1-bbc0ed7586c5: !Template + dcfed41f-b7d2-49c6-90d8-5b6de7043126: !Template answer_choices: A ||| B - id: d11adddf-ae9a-47e1-a7b1-bbc0ed7586c5 + id: dcfed41f-b7d2-49c6-90d8-5b6de7043126 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -281,19 +292,30 @@ templates: original_task: true name: better_A_B reference: '' - dc1309cf-3082-4fcb-86a0-6bccb28b15af: !Template + e045dd82-d3e8-4406-955e-f4a8eb0503b2: !Template answer_choices: Yes ||| No - id: dc1309cf-3082-4fcb-86a0-6bccb28b15af + id: e045dd82-d3e8-4406-955e-f4a8eb0503b2 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_good_yes_no + name: single_sentence_bad_no_yes + reference: '' + f82801b8-7b3d-4b2e-b5c0-2eb0fbded769: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: f82801b8-7b3d-4b2e-b5c0-2eb0fbded769 + jinja: '" ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt reference: '' diff --git a/promptsource/templates/blimp/principle_A_c_command/templates.yaml b/promptsource/templates/blimp/principle_A_c_command/templates.yaml index ab77ebcb6..feed47741 100644 --- a/promptsource/templates/blimp/principle_A_c_command/templates.yaml +++ b/promptsource/templates/blimp/principle_A_c_command/templates.yaml @@ -1,47 +1,36 @@ dataset: blimp subset: principle_A_c_command templates: - 266eacf1-3d82-4adc-bcbd-24a0f45dbaef: !Template - answer_choices: A ||| B - id: 266eacf1-3d82-4adc-bcbd-24a0f45dbaef - jinja: '{% set shuffled_order = [0, 1] | random %} - - Read the following two sentences. - - {% if shuffled_order == 0 %} - - Sentence A: {{ sentence_good }} - - Sentence B: {{ sentence_bad }} - - {% else %} - - Sentence A: {{ sentence_bad }} - - Sentence B: {{ sentence_good }} - - {% endif %} - - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} + 14df57be-0b56-4839-b1c7-7dd1ae33be77: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 14df57be-0b56-4839-b1c7-7dd1ae33be77 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: true_null_prompt + reference: '' + 17b77f2b-6a64-4617-aa7f-fa346bdfe2da: !Template + answer_choices: Yes ||| No + id: 17b77f2b-6a64-4617-aa7f-fa346bdfe2da + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - {% endif %}' + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: better_A_B + original_task: false + name: single_sentence_bad_yes_no reference: '' - 528f6538-e203-431e-99c4-c99c13ec2741: !Template + 224440bb-ad58-44b8-b7e4-9fa911f039d3: !Template answer_choices: 1 ||| 2 - id: 528f6538-e203-431e-99c4-c99c13ec2741 + id: 224440bb-ad58-44b8-b7e4-9fa911f039d3 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -78,30 +67,29 @@ templates: original_task: true name: grammatical_between_1_2 reference: '' - 624031bc-9c70-4dfe-82d1-68ccb8ca0b58: !Template + 3629590c-1902-4af2-806f-e19ef74bf8fd: !Template answer_choices: A ||| B - id: 624031bc-9c70-4dfe-82d1-68ccb8ca0b58 + id: 3629590c-1902-4af2-806f-e19ef74bf8fd jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -115,15 +103,15 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: better_A_B reference: '' - 6df7b6d2-e50d-4386-a836-ab1d627ff06c: !Template + 37b856ed-193d-4add-ac0e-0d5315c0e041: !Template answer_choices: Yes ||| No - id: 6df7b6d2-e50d-4386-a836-ab1d627ff06c + id: 37b856ed-193d-4add-ac0e-0d5315c0e041 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -131,26 +119,37 @@ templates: metrics: - Accuracy original_task: false - name: single_sentence_good_no_yes + name: single_sentence_good_yes_no reference: '' - 878859f7-80d6-49b2-b11f-750b6e945b7b: !Template - answer_choices: A ||| B - id: 878859f7-80d6-49b2-b11f-750b6e945b7b + 6b0c2c8a-373b-4386-a642-b2c208701438: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 6b0c2c8a-373b-4386-a642-b2c208701438 + jinja: '" ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + 94d5681b-fb53-4fd0-9c56-b83806f5beaf: !Template + answer_choices: 1 ||| 2 + id: 94d5681b-fb53-4fd0-9c56-b83806f5beaf jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -170,15 +169,15 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: grammatical_which_one_1_2 reference: '' - 8fcb6fdb-0783-4ec0-88e4-99d525ad09c1: !Template + a44f194a-4937-4425-8164-089dfd4c7c1d: !Template answer_choices: Yes ||| No - id: 8fcb6fdb-0783-4ec0-88e4-99d525ad09c1 + id: a44f194a-4937-4425-8164-089dfd4c7c1d jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -186,11 +185,27 @@ templates: metrics: - Accuracy original_task: false - name: single_sentence_good_yes_no + name: single_sentence_good_no_yes reference: '' - c18476f3-55fa-4381-8ebd-14d7ad7cd4e1: !Template + b78c89c0-72dd-4cdc-939f-63a83d7d83f5: !Template + answer_choices: Yes ||| No + id: b78c89c0-72dd-4cdc-939f-63a83d7d83f5 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes + reference: '' + bf775f9e-0e45-4365-b700-f9c6a4c1ab55: !Template answer_choices: A ||| B - id: c18476f3-55fa-4381-8ebd-14d7ad7cd4e1 + id: bf775f9e-0e45-4365-b700-f9c6a4c1ab55 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -226,24 +241,24 @@ templates: original_task: true name: better_B_A reference: '' - d058331f-baab-4e84-b7ff-fdfc2afb6551: !Template - answer_choices: 1 ||| 2 - id: d058331f-baab-4e84-b7ff-fdfc2afb6551 + c34cc39d-7e16-4151-ba3b-ed09ea2beecc: !Template + answer_choices: A ||| B + id: c34cc39d-7e16-4151-ba3b-ed09ea2beecc jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -263,37 +278,44 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: grammatical_which_one_A_B reference: '' - d673932a-84b4-4e40-90d9-0d3a4b51a25a: !Template - answer_choices: Yes ||| No - id: d673932a-84b4-4e40-90d9-0d3a4b51a25a - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + c7cbe94c-5cd1-4a9a-a3a7-ef7739705c92: !Template + answer_choices: A ||| B + id: c7cbe94c-5cd1-4a9a-a3a7-ef7739705c92 + jinja: '{% set shuffled_order = [0, 1] | random %} - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no - reference: '' - e1e35e99-b548-45f3-b550-58a16e93e127: !Template - answer_choices: Yes ||| No - id: e1e35e99-b548-45f3-b550-58a16e93e127 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + Between the two sentences, which one is grammatical? Please answer A or B. - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: false - name: single_sentence_bad_no_yes + original_task: true + name: grammatical_between_A_B reference: '' diff --git a/promptsource/templates/blimp/principle_A_case_1/templates.yaml b/promptsource/templates/blimp/principle_A_case_1/templates.yaml index d4f98c68c..a1c35e1b8 100644 --- a/promptsource/templates/blimp/principle_A_case_1/templates.yaml +++ b/promptsource/templates/blimp/principle_A_case_1/templates.yaml @@ -1,45 +1,69 @@ dataset: blimp subset: principle_A_case_1 templates: - 3eb3811b-66a1-4628-93fa-a9a73da66986: !Template - answer_choices: Yes ||| No - id: 3eb3811b-66a1-4628-93fa-a9a73da66986 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + 020e3f46-809a-4f54-9a1b-aaebeb10b20a: !Template + answer_choices: 1 ||| 2 + id: 020e3f46-809a-4f54-9a1b-aaebeb10b20a + jinja: '{% set shuffled_order = [0, 1] | random %} - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: false - name: single_sentence_good_no_yes + original_task: true + name: grammatical_which_one_1_2 reference: '' - 44b2b73d-381e-43e4-9f73-dcf282979f5f: !Template + 0b92fd61-691b-448b-8d30-ba1a6f64209b: !Template answer_choices: A ||| B - id: 44b2b73d-381e-43e4-9f73-dcf282979f5f + id: 0b92fd61-691b-448b-8d30-ba1a6f64209b jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -53,11 +77,27 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: grammatical_which_one_A_B + reference: '' + 4639d962-0757-436f-b00c-1d76e78ba1c8: !Template + answer_choices: Yes ||| No + id: 4639d962-0757-436f-b00c-1d76e78ba1c8 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no reference: '' - 6bbdba99-6b3d-4cfa-a38d-274e44c0c67a: !Template + 4868f71e-5558-48b2-ace0-551b620c7164: !Template answer_choices: A ||| B - id: 6bbdba99-6b3d-4cfa-a38d-274e44c0c67a + id: 4868f71e-5558-48b2-ace0-551b620c7164 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -76,7 +116,7 @@ templates: {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order + Which one is a better sentence of English, A or B? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -91,14 +131,30 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: better_A_B + reference: '' + 5a0db8b2-5e16-4df9-a390-a2b495762ba4: !Template + answer_choices: Yes ||| No + id: 5a0db8b2-5e16-4df9-a390-a2b495762ba4 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes reference: '' - 86a16480-2aea-4850-b44e-47d19f9b311d: !Template + 62f2bf9c-15af-42d7-abeb-816d4b7e984c: !Template answer_choices: A ||| B - id: 86a16480-2aea-4850-b44e-47d19f9b311d + id: 62f2bf9c-15af-42d7-abeb-816d4b7e984c jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -130,14 +186,14 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: grammatical_between_A_B reference: '' - a95609b9-6873-451a-90c1-7cfb2cfc8f2c: !Template + 7c697da0-4f1d-407f-9d7b-31417aac1186: !Template answer_choices: 1 ||| 2 - id: a95609b9-6873-451a-90c1-7cfb2cfc8f2c + id: 7c697da0-4f1d-407f-9d7b-31417aac1186 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -169,15 +225,15 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: grammatical_between_1_2 reference: '' - b690d38b-d4d2-46c6-98d8-030f7c36e500: !Template + 8292012a-5704-4343-97cb-83bb02f628ff: !Template answer_choices: Yes ||| No - id: b690d38b-d4d2-46c6-98d8-030f7c36e500 + id: 8292012a-5704-4343-97cb-83bb02f628ff jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -185,87 +241,53 @@ templates: metrics: - Accuracy original_task: false - name: single_sentence_good_yes_no + name: single_sentence_good_no_yes reference: '' - bbdf5636-0581-48d2-8929-354f4dc35776: !Template - answer_choices: Yes ||| No - id: bbdf5636-0581-48d2-8929-354f4dc35776 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' + 9ec45076-a10c-462b-96e5-b5c43012f16c: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 9ec45076-a10c-462b-96e5-b5c43012f16c + jinja: ' ||| {{ sentence_good }}' metadata: !TemplateMetadata - choices_in_prompt: true + choices_in_prompt: false metrics: - Accuracy - original_task: false - name: single_sentence_bad_no_yes + original_task: true + name: true_null_prompt reference: '' - f2238a93-85a7-4ad7-b959-d9600b403a9b: !Template - answer_choices: 1 ||| 2 - id: f2238a93-85a7-4ad7-b959-d9600b403a9b - jinja: '{% set shuffled_order = [0, 1] | random %} - - Between the two sentences, which one is grammatical? Please answer 1 or 2. - - {% if shuffled_order == 0 %} - - 1: {{ sentence_good }} - - 2: {{ sentence_bad }} - - {% else %} - - 1: {{ sentence_bad }} - - 2: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} - - {% endif %}' + b1bac90e-bfd2-4cc5-8f5e-0677e2460abc: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: b1bac90e-bfd2-4cc5-8f5e-0677e2460abc + jinja: '" ||| {{ sentence_good }}' metadata: !TemplateMetadata - choices_in_prompt: true + choices_in_prompt: false metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: null_prompt reference: '' - f30b527e-273e-46ff-9604-ff050fa978fb: !Template + fbfcc0fb-2955-4d87-9ace-0bd752a71378: !Template answer_choices: A ||| B - id: f30b527e-273e-46ff-9604-ff050fa978fb + id: fbfcc0fb-2955-4d87-9ace-0bd752a71378 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -279,11 +301,11 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: better_B_A reference: '' - fb1dd699-4d2f-462b-9365-2e0dc323cdad: !Template + fe227836-7a7e-41af-8fa8-6505a70eb3b9: !Template answer_choices: Yes ||| No - id: fb1dd699-4d2f-462b-9365-2e0dc323cdad + id: fe227836-7a7e-41af-8fa8-6505a70eb3b9 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" diff --git a/promptsource/templates/blimp/principle_A_case_2/templates.yaml b/promptsource/templates/blimp/principle_A_case_2/templates.yaml index e1770bbc2..00621d338 100644 --- a/promptsource/templates/blimp/principle_A_case_2/templates.yaml +++ b/promptsource/templates/blimp/principle_A_case_2/templates.yaml @@ -1,47 +1,41 @@ dataset: blimp subset: principle_A_case_2 templates: - 07a2c2fe-d131-4e00-b06e-c9f944ebc667: !Template - answer_choices: A ||| B - id: 07a2c2fe-d131-4e00-b06e-c9f944ebc667 - jinja: '{% set shuffled_order = [0, 1] | random %} - - Read the following two sentences. - - {% if shuffled_order == 0 %} - - Sentence A: {{ sentence_good }} - - Sentence B: {{ sentence_bad }} - - {% else %} - - Sentence A: {{ sentence_bad }} - - Sentence B: {{ sentence_good }} - - {% endif %} - - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} - - {{ answer_choices[0] }} - - {% else %} + 1b97a778-bb48-4ac1-bc2c-606a14bda5a5: !Template + answer_choices: Yes ||| No + id: 1b97a778-bb48-4ac1-bc2c-606a14bda5a5 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - {{ answer_choices[1] }} + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 2111ed47-d96c-46e1-a88d-87e41fcaaf0c: !Template + answer_choices: Yes ||| No + id: 2111ed47-d96c-46e1-a88d-87e41fcaaf0c + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - {% endif %}' + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: better_A_B + original_task: false + name: single_sentence_bad_no_yes reference: '' - 1be73e9c-bd8e-46e6-b0ab-a3950aebafe6: !Template + 21f07345-80de-4366-86d4-7cfa00c2c951: !Template answer_choices: A ||| B - id: 1be73e9c-bd8e-46e6-b0ab-a3950aebafe6 + id: 21f07345-80de-4366-86d4-7cfa00c2c951 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -77,24 +71,24 @@ templates: original_task: true name: better_B_A reference: '' - 42f048db-e7cb-4984-9301-d703de4668ee: !Template - answer_choices: A ||| B - id: 42f048db-e7cb-4984-9301-d703de4668ee + 2217a186-ab74-42d7-bd74-4c17d3cda486: !Template + answer_choices: 1 ||| 2 + id: 2217a186-ab74-42d7-bd74-4c17d3cda486 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -114,27 +108,38 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: grammatical_which_one_1_2 reference: '' - 4b17e2fe-920a-49c3-8ce6-417c8d3de8f4: !Template + 33d2aaa7-0ee4-4afc-b383-8cba8d34f2b6: !Template answer_choices: Yes ||| No - id: 4b17e2fe-920a-49c3-8ce6-417c8d3de8f4 + id: 33d2aaa7-0ee4-4afc-b383-8cba8d34f2b6 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_bad_no_yes + name: single_sentence_good_no_yes reference: '' - 7cf6d5b8-f88f-4dee-80d0-4b839cbe4ff7: !Template + 3d95a90a-ed9e-4d7e-b486-fc6cbaf8c317: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 3d95a90a-ed9e-4d7e-b486-fc6cbaf8c317 + jinja: '" ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + 3dc3e400-c463-4d49-96f2-cf863cf459de: !Template answer_choices: 1 ||| 2 - id: 7cf6d5b8-f88f-4dee-80d0-4b839cbe4ff7 + id: 3dc3e400-c463-4d49-96f2-cf863cf459de jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -171,62 +176,68 @@ templates: original_task: true name: grammatical_between_1_2 reference: '' - 8d2a603b-c0ab-455d-a8fa-f36f452a1d7b: !Template - answer_choices: Yes ||| No - id: 8d2a603b-c0ab-455d-a8fa-f36f452a1d7b - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + 54811f76-307b-4685-a3c3-f747f6dfd7a3: !Template + answer_choices: A ||| B + id: 54811f76-307b-4685-a3c3-f747f6dfd7a3 + jinja: '{% set shuffled_order = [0, 1] | random %} - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no - reference: '' - ab66f18b-bcc9-4256-9a4e-dd71b7b5193f: !Template - answer_choices: Yes ||| No - id: ab66f18b-bcc9-4256-9a4e-dd71b7b5193f - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + Between the two sentences, which one is grammatical? Please answer A or B. - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: false - name: single_sentence_good_no_yes + original_task: true + name: grammatical_between_A_B reference: '' - b13277a5-1402-41d5-af0e-fb9918efd902: !Template - answer_choices: 1 ||| 2 - id: b13277a5-1402-41d5-af0e-fb9918efd902 + bea83acb-6c7c-499e-b51d-866db4df97f8: !Template + answer_choices: A ||| B + id: bea83acb-6c7c-499e-b51d-866db4df97f8 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -240,11 +251,22 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: better_A_B reference: '' - c1d7dd5b-a830-4373-b9a9-70359f89ec40: !Template + cc388e7e-bc66-43e0-b964-8be7ceff81fa: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: cc388e7e-bc66-43e0-b964-8be7ceff81fa + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: true_null_prompt + reference: '' + d5bdbca8-6eb9-4bb7-90af-254e3eb29cf4: !Template answer_choices: Yes ||| No - id: c1d7dd5b-a830-4373-b9a9-70359f89ec40 + id: d5bdbca8-6eb9-4bb7-90af-254e3eb29cf4 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -258,12 +280,12 @@ templates: original_task: false name: single_sentence_bad_yes_no reference: '' - eefd5af0-3c31-434d-8fa7-063247b47d04: !Template + e05d659e-9c31-4e16-af3b-76d90b249f65: !Template answer_choices: A ||| B - id: eefd5af0-3c31-434d-8fa7-063247b47d04 + id: e05d659e-9c31-4e16-af3b-76d90b249f65 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -295,5 +317,5 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: grammatical_which_one_A_B reference: '' diff --git a/promptsource/templates/blimp/principle_A_domain_1/templates.yaml b/promptsource/templates/blimp/principle_A_domain_1/templates.yaml index ed9298fa3..ae3e998f7 100644 --- a/promptsource/templates/blimp/principle_A_domain_1/templates.yaml +++ b/promptsource/templates/blimp/principle_A_domain_1/templates.yaml @@ -1,30 +1,29 @@ dataset: blimp subset: principle_A_domain_1 templates: - 3b145aa4-2436-487b-a7dc-8b5897b6fb41: !Template + 1189a014-98f3-4ade-b109-54b8d1f848d7: !Template answer_choices: A ||| B - id: 3b145aa4-2436-487b-a7dc-8b5897b6fb41 + id: 1189a014-98f3-4ade-b109-54b8d1f848d7 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -38,27 +37,11 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B - reference: '' - 3f29d99e-8bef-477e-935e-faf920c15efe: !Template - answer_choices: Yes ||| No - id: 3f29d99e-8bef-477e-935e-faf920c15efe - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes + name: better_A_B reference: '' - 4998cbe8-a5b7-4055-acc0-12cb44a85cc3: !Template + 20639897-58c5-490d-9267-f151776ac38c: !Template answer_choices: A ||| B - id: 4998cbe8-a5b7-4055-acc0-12cb44a85cc3 + id: 20639897-58c5-490d-9267-f151776ac38c jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -94,60 +77,71 @@ templates: original_task: true name: better_B_A reference: '' - 4b2276f4-a44a-42af-871d-d4e4956cf8c2: !Template + a5fa2d01-a62a-4adf-b59a-cf5e11e62557: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: a5fa2d01-a62a-4adf-b59a-cf5e11e62557 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: true_null_prompt + reference: '' + b8637d62-51cd-4042-8bc6-39327c4d8b77: !Template answer_choices: Yes ||| No - id: 4b2276f4-a44a-42af-871d-d4e4956cf8c2 + id: b8637d62-51cd-4042-8bc6-39327c4d8b77 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_bad_yes_no + name: single_sentence_good_yes_no reference: '' - 84ad15ba-205b-451c-95a6-2adde1c712b1: !Template + c9292620-ea8a-4df6-94c5-c41965971393: !Template answer_choices: Yes ||| No - id: 84ad15ba-205b-451c-95a6-2adde1c712b1 + id: c9292620-ea8a-4df6-94c5-c41965971393 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_good_yes_no + name: single_sentence_bad_no_yes reference: '' - 9ff72b55-5376-4f53-84dc-ea2054350a8c: !Template + ccc246e2-3c4a-474f-a24d-e5cb9851998a: !Template answer_choices: Yes ||| No - id: 9ff72b55-5376-4f53-84dc-ea2054350a8c + id: ccc246e2-3c4a-474f-a24d-e5cb9851998a jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_bad_no_yes + name: single_sentence_good_no_yes reference: '' - ca28bbd1-7e74-474f-a905-6232fa7ba6fa: !Template + d19145e1-cd40-41d6-9f65-08a9cbd25595: !Template answer_choices: A ||| B - id: ca28bbd1-7e74-474f-a905-6232fa7ba6fa + id: d19145e1-cd40-41d6-9f65-08a9cbd25595 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -179,31 +173,43 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: grammatical_between_A_B + reference: '' + da416f81-40e7-4f21-a86a-b72587fdc375: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: da416f81-40e7-4f21-a86a-b72587fdc375 + jinja: '" ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt reference: '' - d65a7429-4400-4758-9dcb-02340e62f7e5: !Template + daa889ef-9eec-451c-ae3b-9c1a19b08399: !Template answer_choices: A ||| B - id: d65a7429-4400-4758-9dcb-02340e62f7e5 + id: daa889ef-9eec-451c-ae3b-9c1a19b08399 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -217,11 +223,11 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: grammatical_which_one_A_B reference: '' - e3f52395-7418-4d01-845d-d651b87e9a96: !Template + dd7e3e25-08d2-4f46-ba4b-bd0689bbb8c9: !Template answer_choices: 1 ||| 2 - id: e3f52395-7418-4d01-845d-d651b87e9a96 + id: dd7e3e25-08d2-4f46-ba4b-bd0689bbb8c9 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -258,9 +264,9 @@ templates: original_task: true name: grammatical_between_1_2 reference: '' - eee44046-fd19-4476-a6b8-d1f66537f282: !Template + e1e5ea8d-4fda-4a50-bfe9-1939c062200b: !Template answer_choices: 1 ||| 2 - id: eee44046-fd19-4476-a6b8-d1f66537f282 + id: e1e5ea8d-4fda-4a50-bfe9-1939c062200b jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer 1 or 2. @@ -297,3 +303,19 @@ templates: original_task: true name: grammatical_which_one_1_2 reference: '' + f07bbec3-f3f4-4328-ad11-0e0f55570c5b: !Template + answer_choices: Yes ||| No + id: f07bbec3-f3f4-4328-ad11-0e0f55570c5b + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' diff --git a/promptsource/templates/blimp/principle_A_domain_2/templates.yaml b/promptsource/templates/blimp/principle_A_domain_2/templates.yaml index e6611417c..5107135d0 100644 --- a/promptsource/templates/blimp/principle_A_domain_2/templates.yaml +++ b/promptsource/templates/blimp/principle_A_domain_2/templates.yaml @@ -1,24 +1,40 @@ dataset: blimp subset: principle_A_domain_2 templates: - 0d84877a-7c75-4802-9295-4474c6aa0748: !Template - answer_choices: A ||| B - id: 0d84877a-7c75-4802-9295-4474c6aa0748 + 17efc25a-f1d3-4c30-9808-b53e024866fc: !Template + answer_choices: Yes ||| No + id: 17efc25a-f1d3-4c30-9808-b53e024866fc + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes + reference: '' + 1cf30ce6-2854-4dd6-ae35-a4d7d172b056: !Template + answer_choices: 1 ||| 2 + id: 1cf30ce6-2854-4dd6-ae35-a4d7d172b056 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -38,43 +54,11 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B - reference: '' - 439f4802-1aec-4191-813f-6477f2dda250: !Template - answer_choices: Yes ||| No - id: 439f4802-1aec-4191-813f-6477f2dda250 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes - reference: '' - 4dc27764-1806-440e-8fe0-6dbf45ba10e0: !Template - answer_choices: Yes ||| No - id: 4dc27764-1806-440e-8fe0-6dbf45ba10e0 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no + name: grammatical_between_1_2 reference: '' - 6b97560e-878b-41ae-a94d-a70eb631ffe7: !Template + 39999513-7266-47a4-be66-7eb3b2ca4d18: !Template answer_choices: 1 ||| 2 - id: 6b97560e-878b-41ae-a94d-a70eb631ffe7 + id: 39999513-7266-47a4-be66-7eb3b2ca4d18 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer 1 or 2. @@ -111,29 +95,100 @@ templates: original_task: true name: grammatical_which_one_1_2 reference: '' - 78a064b5-081c-4383-a0ed-4620e9f586c8: !Template + 521b74bd-8322-4e95-85df-41ccce4c92f8: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 521b74bd-8322-4e95-85df-41ccce4c92f8 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: true_null_prompt + reference: '' + 56c62ec2-f6d6-480d-a7d4-79d81eb155b2: !Template + answer_choices: Yes ||| No + id: 56c62ec2-f6d6-480d-a7d4-79d81eb155b2 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 8ac81655-6870-407b-a04c-d2a74c0c2ef4: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 8ac81655-6870-407b-a04c-d2a74c0c2ef4 + jinja: '" ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + 92aeb196-9316-4094-99eb-ac6e9dff998f: !Template + answer_choices: Yes ||| No + id: 92aeb196-9316-4094-99eb-ac6e9dff998f + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + a33a14dd-d063-40af-9c44-c80aa7ad995f: !Template + answer_choices: Yes ||| No + id: a33a14dd-d063-40af-9c44-c80aa7ad995f + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes + reference: '' + af3910c8-2853-4604-b0fe-916334c67762: !Template answer_choices: A ||| B - id: 78a064b5-081c-4383-a0ed-4620e9f586c8 + id: af3910c8-2853-4604-b0fe-916334c67762 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -147,26 +202,26 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: grammatical_between_A_B reference: '' - 9b8aa47f-d848-4ac3-b7eb-d0c24def596a: !Template - answer_choices: 1 ||| 2 - id: 9b8aa47f-d848-4ac3-b7eb-d0c24def596a + b08ef95e-cf26-41b4-9656-51a4a1789e5c: !Template + answer_choices: A ||| B + id: b08ef95e-cf26-41b4-9656-51a4a1789e5c jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -186,11 +241,11 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: grammatical_which_one_A_B reference: '' - 9c63204c-ea2d-45a6-a1c8-fb0ed98f97eb: !Template + cd6fa422-0f37-4049-83e2-f471b70e4b18: !Template answer_choices: A ||| B - id: 9c63204c-ea2d-45a6-a1c8-fb0ed98f97eb + id: cd6fa422-0f37-4049-83e2-f471b70e4b18 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -209,7 +264,7 @@ templates: {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order + Which one is a better sentence of English, A or B? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -224,64 +279,31 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A - reference: '' - a8e59cea-16ba-406e-a634-5d73f688c99e: !Template - answer_choices: Yes ||| No - id: a8e59cea-16ba-406e-a634-5d73f688c99e - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no - reference: '' - b8f06ed1-ee37-42f3-a150-01f6bb13ea9e: !Template - answer_choices: Yes ||| No - id: b8f06ed1-ee37-42f3-a150-01f6bb13ea9e - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes + name: better_A_B reference: '' - c09fb1af-64d5-4070-954c-0b80ea07943e: !Template + e3495bf0-f8a4-488b-925f-1a0abdde5d6d: !Template answer_choices: A ||| B - id: c09fb1af-64d5-4070-954c-0b80ea07943e + id: e3495bf0-f8a4-488b-925f-1a0abdde5d6d jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -295,5 +317,5 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: better_B_A reference: '' diff --git a/promptsource/templates/blimp/principle_A_domain_3/templates.yaml b/promptsource/templates/blimp/principle_A_domain_3/templates.yaml index 0a48a8159..cb964877f 100644 --- a/promptsource/templates/blimp/principle_A_domain_3/templates.yaml +++ b/promptsource/templates/blimp/principle_A_domain_3/templates.yaml @@ -1,24 +1,24 @@ dataset: blimp subset: principle_A_domain_3 templates: - 07eccea8-220d-4c16-8e6e-68238346309b: !Template - answer_choices: 1 ||| 2 - id: 07eccea8-220d-4c16-8e6e-68238346309b + 03fe1aff-95fe-49a1-ab6e-5c4c16b22181: !Template + answer_choices: A ||| B + id: 03fe1aff-95fe-49a1-ab6e-5c4c16b22181 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -38,43 +38,11 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 - reference: '' - 630d4dc2-1bfe-428d-af74-2a0d084627b2: !Template - answer_choices: Yes ||| No - id: 630d4dc2-1bfe-428d-af74-2a0d084627b2 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes - reference: '' - 6427fbe6-0837-49ed-88e0-95bb0f100bb3: !Template - answer_choices: Yes ||| No - id: 6427fbe6-0837-49ed-88e0-95bb0f100bb3 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no + name: grammatical_between_A_B reference: '' - 68ed5a47-46e3-451c-8df1-1c8dd7bce47f: !Template + 13a812d9-c450-40bd-9a25-481208c79b90: !Template answer_choices: A ||| B - id: 68ed5a47-46e3-451c-8df1-1c8dd7bce47f + id: 13a812d9-c450-40bd-9a25-481208c79b90 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer A or B. @@ -111,12 +79,44 @@ templates: original_task: true name: grammatical_which_one_A_B reference: '' - 7c3be9e9-98ee-4e72-a714-083ced37debf: !Template + 146f4806-61a9-43ff-8ac4-ff7f9f5dcc2c: !Template + answer_choices: Yes ||| No + id: 146f4806-61a9-43ff-8ac4-ff7f9f5dcc2c + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes + reference: '' + 25f4d436-274a-4b53-a2d7-63b80bec29c4: !Template + answer_choices: Yes ||| No + id: 25f4d436-274a-4b53-a2d7-63b80bec29c4 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 3f900e5b-df69-459d-b305-364aa5442c77: !Template answer_choices: 1 ||| 2 - id: 7c3be9e9-98ee-4e72-a714-083ced37debf + id: 3f900e5b-df69-459d-b305-364aa5442c77 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -148,48 +148,47 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: grammatical_between_1_2 reference: '' - 8ea6b960-cfcd-4acd-af69-03b9a1a47252: !Template + 417732f2-c7e3-4984-84f5-4f6c97027b67: !Template answer_choices: Yes ||| No - id: 8ea6b960-cfcd-4acd-af69-03b9a1a47252 + id: 417732f2-c7e3-4984-84f5-4f6c97027b67 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_bad_no_yes + name: single_sentence_good_yes_no reference: '' - 950e78ff-023c-401e-9866-ddacc2d10820: !Template + 57f604f4-f765-4362-908a-251b0c5fa157: !Template answer_choices: A ||| B - id: 950e78ff-023c-401e-9866-ddacc2d10820 + id: 57f604f4-f765-4362-908a-251b0c5fa157 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -203,47 +202,32 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B - reference: '' - bd438d35-ae7e-4fca-978a-02d990d3bbf8: !Template - answer_choices: Yes ||| No - id: bd438d35-ae7e-4fca-978a-02d990d3bbf8 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no + name: better_A_B reference: '' - c4a7f3eb-c4f5-4dac-bc0c-d6dda5da77ef: !Template - answer_choices: A ||| B - id: c4a7f3eb-c4f5-4dac-bc0c-d6dda5da77ef + 57fb6288-488c-42af-b259-681edc6222a7: !Template + answer_choices: 1 ||| 2 + id: 57fb6288-488c-42af-b259-681edc6222a7 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -257,11 +241,11 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: grammatical_which_one_1_2 reference: '' - c5f8ee18-8137-4af8-b086-6330963739f9: !Template + 683faa0d-31fc-465f-ad84-da5a62b83a83: !Template answer_choices: A ||| B - id: c5f8ee18-8137-4af8-b086-6330963739f9 + id: 683faa0d-31fc-465f-ad84-da5a62b83a83 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -280,7 +264,7 @@ templates: {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order + Which one is a better sentence of English, B or A? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -295,5 +279,43 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: better_B_A + reference: '' + 77c5d27f-4438-4fd2-a1a8-896da83de7d0: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 77c5d27f-4438-4fd2-a1a8-896da83de7d0 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: true_null_prompt + reference: '' + 846c2936-6e97-43a9-9e4a-1fe40387bffb: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 846c2936-6e97-43a9-9e4a-1fe40387bffb + jinja: '" ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + a56dac22-38e6-4752-873a-51d024298c21: !Template + answer_choices: Yes ||| No + id: a56dac22-38e6-4752-873a-51d024298c21 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes reference: '' diff --git a/promptsource/templates/blimp/principle_A_reconstruction/templates.yaml b/promptsource/templates/blimp/principle_A_reconstruction/templates.yaml index 17583ced8..80382b3bd 100644 --- a/promptsource/templates/blimp/principle_A_reconstruction/templates.yaml +++ b/promptsource/templates/blimp/principle_A_reconstruction/templates.yaml @@ -1,9 +1,9 @@ dataset: blimp subset: principle_A_reconstruction templates: - 255e6923-6c36-4bbb-86a6-3e9aaa1f048a: !Template + 00394236-354b-4697-b870-a3acb0232856: !Template answer_choices: Yes ||| No - id: 255e6923-6c36-4bbb-86a6-3e9aaa1f048a + id: 00394236-354b-4697-b870-a3acb0232856 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -17,45 +17,30 @@ templates: original_task: false name: single_sentence_bad_yes_no reference: '' - 28235ddb-4ecb-41d4-b571-179ef4be2934: !Template - answer_choices: Yes ||| No - id: 28235ddb-4ecb-41d4-b571-179ef4be2934 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes - reference: '' - 299bf6ee-117a-4e38-91b2-6c3fe9043f85: !Template + 15ca5428-e8ab-4c09-93d4-29fb6dfccc65: !Template answer_choices: A ||| B - id: 299bf6ee-117a-4e38-91b2-6c3fe9043f85 + id: 15ca5428-e8ab-4c09-93d4-29fb6dfccc65 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -69,48 +54,31 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A - reference: '' - 3c781311-5236-4805-b6f1-90bb44da84cd: !Template - answer_choices: Yes ||| No - id: 3c781311-5236-4805-b6f1-90bb44da84cd - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes + name: grammatical_between_A_B reference: '' - 3d6382bf-026b-4448-8c14-ee7296731e48: !Template + 1d7f7749-e21b-42ae-a7f2-03a378f1dd77: !Template answer_choices: A ||| B - id: 3d6382bf-026b-4448-8c14-ee7296731e48 + id: 1d7f7749-e21b-42ae-a7f2-03a378f1dd77 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -124,11 +92,11 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: better_A_B reference: '' - 3e90b2cb-0b87-446a-9ff7-e6f0378101c8: !Template + 427171c8-e1c9-4c97-a55d-ab7757bdd0c8: !Template answer_choices: 1 ||| 2 - id: 3e90b2cb-0b87-446a-9ff7-e6f0378101c8 + id: 427171c8-e1c9-4c97-a55d-ab7757bdd0c8 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer 1 or 2. @@ -165,30 +133,29 @@ templates: original_task: true name: grammatical_which_one_1_2 reference: '' - 902acbd1-3493-4ea3-8cef-fa8942784645: !Template + 4bd7341a-5539-4afb-a791-92744dc256d5: !Template answer_choices: A ||| B - id: 902acbd1-3493-4ea3-8cef-fa8942784645 + id: 4bd7341a-5539-4afb-a791-92744dc256d5 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -202,31 +169,48 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: better_B_A reference: '' - bd2bca31-aadd-4544-b8e8-7519cb1438f5: !Template - answer_choices: A ||| B - id: bd2bca31-aadd-4544-b8e8-7519cb1438f5 + 56342922-df71-4cfe-8424-560b1c2a7a24: !Template + answer_choices: Yes ||| No + id: 56342922-df71-4cfe-8424-560b1c2a7a24 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes + reference: '' + 6ae9a244-bfaa-443b-a0d0-34856b47fd68: !Template + answer_choices: 1 ||| 2 + id: 6ae9a244-bfaa-443b-a0d0-34856b47fd68 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -240,42 +224,37 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: grammatical_between_1_2 reference: '' - bf05357a-eac9-46b1-8063-614519e87174: !Template - answer_choices: Yes ||| No - id: bf05357a-eac9-46b1-8063-614519e87174 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + 96921ce7-cc0b-40ea-ae48-d28ead268e8c: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 96921ce7-cc0b-40ea-ae48-d28ead268e8c + jinja: '" ||| {{ sentence_good }}' metadata: !TemplateMetadata - choices_in_prompt: true + choices_in_prompt: false metrics: - Accuracy - original_task: false - name: single_sentence_good_yes_no + original_task: true + name: null_prompt reference: '' - d4734cfb-f96d-418e-9def-e0a2beccdb33: !Template - answer_choices: 1 ||| 2 - id: d4734cfb-f96d-418e-9def-e0a2beccdb33 + 9a3b8431-b8fc-49a3-9dd5-5de7c8f516f7: !Template + answer_choices: A ||| B + id: 9a3b8431-b8fc-49a3-9dd5-5de7c8f516f7 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -295,5 +274,48 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: grammatical_which_one_A_B + reference: '' + c0213958-4f02-40c4-8b9e-9ba412d609dd: !Template + answer_choices: Yes ||| No + id: c0213958-4f02-40c4-8b9e-9ba412d609dd + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + c631546b-227c-4a99-a94f-f8420c4a1312: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: c631546b-227c-4a99-a94f-f8420c4a1312 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: true_null_prompt + reference: '' + ef661aa7-70a7-472b-b9a8-7e8188274704: !Template + answer_choices: Yes ||| No + id: ef661aa7-70a7-472b-b9a8-7e8188274704 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes reference: '' diff --git a/promptsource/templates/blimp/regular_plural_subject_verb_agreement_1/templates.yaml b/promptsource/templates/blimp/regular_plural_subject_verb_agreement_1/templates.yaml index 23e6a168e..45a94b975 100644 --- a/promptsource/templates/blimp/regular_plural_subject_verb_agreement_1/templates.yaml +++ b/promptsource/templates/blimp/regular_plural_subject_verb_agreement_1/templates.yaml @@ -1,62 +1,40 @@ dataset: blimp subset: regular_plural_subject_verb_agreement_1 templates: - 03829374-4f10-4144-91cb-16eb4a00495d: !Template - answer_choices: Yes ||| No - id: 03829374-4f10-4144-91cb-16eb4a00495d - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no - reference: '' - 1eae7b0b-42e9-4fea-9270-e35cb3280f23: !Template - answer_choices: Yes ||| No - id: 1eae7b0b-42e9-4fea-9270-e35cb3280f23 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + 1014056b-4114-4213-85bf-9718961144ad: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 1014056b-4114-4213-85bf-9718961144ad + jinja: '" ||| {{ sentence_good }}' metadata: !TemplateMetadata - choices_in_prompt: true + choices_in_prompt: false metrics: - Accuracy - original_task: false - name: single_sentence_good_yes_no + original_task: true + name: null_prompt reference: '' - 26fa366b-d7d5-42bb-b5f2-7c58135286d3: !Template - answer_choices: 1 ||| 2 - id: 26fa366b-d7d5-42bb-b5f2-7c58135286d3 + 11a581f6-aa83-4235-8840-4d8348b9f9ec: !Template + answer_choices: A ||| B + id: 11a581f6-aa83-4235-8840-4d8348b9f9ec jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -70,11 +48,11 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: better_B_A reference: '' - 3a5e64f9-3ae3-4104-a0a4-0f92a6a9a5c9: !Template + 11f69f2f-e468-47f5-8910-13c9e463b357: !Template answer_choices: A ||| B - id: 3a5e64f9-3ae3-4104-a0a4-0f92a6a9a5c9 + id: 11f69f2f-e468-47f5-8910-13c9e463b357 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -93,7 +71,7 @@ templates: {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order + Which one is a better sentence of English, A or B? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -108,14 +86,14 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: better_A_B reference: '' - 5c7da74e-fb56-4846-950d-a0a934751260: !Template + 30935e75-402b-432f-b81a-02841ae03398: !Template answer_choices: 1 ||| 2 - id: 5c7da74e-fb56-4846-950d-a0a934751260 + id: 30935e75-402b-432f-b81a-02841ae03398 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -147,14 +125,30 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: grammatical_which_one_1_2 + reference: '' + 686309dd-f772-462a-be99-e6c3af95ddd2: !Template + answer_choices: Yes ||| No + id: 686309dd-f772-462a-be99-e6c3af95ddd2 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes reference: '' - 60c569bb-0814-4ece-9c68-78982b58c812: !Template + 8fa5dae0-cd60-4e35-9beb-ac5e5b49e513: !Template answer_choices: A ||| B - id: 60c569bb-0814-4ece-9c68-78982b58c812 + id: 8fa5dae0-cd60-4e35-9beb-ac5e5b49e513 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -186,26 +180,85 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: grammatical_between_A_B reference: '' - a95dfe01-aa62-4bbc-a133-5c629d0a8f13: !Template - answer_choices: A ||| B - id: a95dfe01-aa62-4bbc-a133-5c629d0a8f13 + 93384294-5451-41de-8c2f-4b2937d16d14: !Template + answer_choices: Yes ||| No + id: 93384294-5451-41de-8c2f-4b2937d16d14 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes + reference: '' + 98a04913-0d5e-4e10-96b4-b8529d34c4ee: !Template + answer_choices: Yes ||| No + id: 98a04913-0d5e-4e10-96b4-b8529d34c4ee + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + b7f37976-d243-4fda-98ad-3024e56fd3e9: !Template + answer_choices: Yes ||| No + id: b7f37976-d243-4fda-98ad-3024e56fd3e9 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + e95f907b-9301-46c3-a24b-90f675831f5e: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: e95f907b-9301-46c3-a24b-90f675831f5e + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: true_null_prompt + reference: '' + eb950807-9957-4685-a205-8f9280cb6961: !Template + answer_choices: 1 ||| 2 + id: eb950807-9957-4685-a205-8f9280cb6961 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -225,31 +278,32 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: grammatical_between_1_2 reference: '' - b2642c64-3fdf-4cd9-a36a-add92f0f0b25: !Template + fed0ceb0-61d5-4e9f-af5d-8268de202a15: !Template answer_choices: A ||| B - id: b2642c64-3fdf-4cd9-a36a-add92f0f0b25 + id: fed0ceb0-61d5-4e9f-af5d-8268de202a15 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -263,37 +317,5 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B - reference: '' - e1c58f46-2f51-4e1c-b4ca-99d8b1df59ec: !Template - answer_choices: Yes ||| No - id: e1c58f46-2f51-4e1c-b4ca-99d8b1df59ec - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes - reference: '' - e48d87f9-47f2-44ab-9c5f-4a1c05e48d1e: !Template - answer_choices: Yes ||| No - id: e48d87f9-47f2-44ab-9c5f-4a1c05e48d1e - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes + name: grammatical_which_one_A_B reference: '' diff --git a/promptsource/templates/blimp/regular_plural_subject_verb_agreement_2/templates.yaml b/promptsource/templates/blimp/regular_plural_subject_verb_agreement_2/templates.yaml index a28497131..a2a1d6552 100644 --- a/promptsource/templates/blimp/regular_plural_subject_verb_agreement_2/templates.yaml +++ b/promptsource/templates/blimp/regular_plural_subject_verb_agreement_2/templates.yaml @@ -1,48 +1,9 @@ dataset: blimp subset: regular_plural_subject_verb_agreement_2 templates: - 03ee8117-9f09-409c-89cb-0d5901a6fa42: !Template - answer_choices: 1 ||| 2 - id: 03ee8117-9f09-409c-89cb-0d5901a6fa42 - jinja: '{% set shuffled_order = [0, 1] | random %} - - Which one of the following sentences is grammatical? Please answer 1 or 2. - - {% if shuffled_order == 0 %} - - 1: {{ sentence_good }} - - 2: {{ sentence_bad }} - - {% else %} - - 1: {{ sentence_bad }} - - 2: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} - - {% endif %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: grammatical_which_one_1_2 - reference: '' - 09ddee17-3c41-4343-85d5-78f3766deb52: !Template + 1a3280a2-fc7d-4e83-9f9f-7e2f296ba77a: !Template answer_choices: A ||| B - id: 09ddee17-3c41-4343-85d5-78f3766deb52 + id: 1a3280a2-fc7d-4e83-9f9f-7e2f296ba77a jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -78,28 +39,12 @@ templates: original_task: true name: better_A_B reference: '' - 3807b338-f2b9-437f-8d96-c3b045cad368: !Template - answer_choices: Yes ||| No - id: 3807b338-f2b9-437f-8d96-c3b045cad368 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no - reference: '' - 5474d660-bfd3-40d0-ab81-8ba59b0d9763: !Template + 2d45742a-6652-493f-abab-cd92bc638151: !Template answer_choices: A ||| B - id: 5474d660-bfd3-40d0-ab81-8ba59b0d9763 + id: 2d45742a-6652-493f-abab-cd92bc638151 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -131,11 +76,27 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: grammatical_which_one_A_B reference: '' - 87c5d723-7a5f-4764-8a79-099dea11505f: !Template + 3346144c-78a2-41e9-97b6-3150c25f55bc: !Template + answer_choices: Yes ||| No + id: 3346144c-78a2-41e9-97b6-3150c25f55bc + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 69538248-50fb-4b7f-bb30-0902152a9a33: !Template answer_choices: A ||| B - id: 87c5d723-7a5f-4764-8a79-099dea11505f + id: 69538248-50fb-4b7f-bb30-0902152a9a33 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -171,9 +132,25 @@ templates: original_task: true name: better_B_A reference: '' - a17707c6-a358-4e15-89b0-4451c6a7e800: !Template + 6cdcf461-ef6c-4a1d-a039-3709a6730f7d: !Template + answer_choices: Yes ||| No + id: 6cdcf461-ef6c-4a1d-a039-3709a6730f7d + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 6f178a7e-43b4-4ebe-98fe-58a4dc61e430: !Template answer_choices: Yes ||| No - id: a17707c6-a358-4e15-89b0-4451c6a7e800 + id: 6f178a7e-43b4-4ebe-98fe-58a4dc61e430 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -187,12 +164,50 @@ templates: original_task: false name: single_sentence_bad_no_yes reference: '' - b71b2409-7828-4888-8577-87f09a67b414: !Template + 6fc23973-257b-445f-9ca6-b341240eaae5: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 6fc23973-257b-445f-9ca6-b341240eaae5 + jinja: '" ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + 7b829592-2c44-45c2-a092-1165f9012e1a: !Template + answer_choices: Yes ||| No + id: 7b829592-2c44-45c2-a092-1165f9012e1a + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes + reference: '' + 9fbf1297-64a5-403a-bb3b-2799a0cae9fc: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 9fbf1297-64a5-403a-bb3b-2799a0cae9fc + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: true_null_prompt + reference: '' + d14843af-6477-451e-814f-c0e50112732b: !Template answer_choices: 1 ||| 2 - id: b71b2409-7828-4888-8577-87f09a67b414 + id: d14843af-6477-451e-814f-c0e50112732b jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -224,14 +239,14 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: grammatical_which_one_1_2 reference: '' - cc6fb3df-3728-4ba7-bd87-a4f92c22aebf: !Template + d2c61505-b860-4812-bb63-ebcb015a226b: !Template answer_choices: A ||| B - id: cc6fb3df-3728-4ba7-bd87-a4f92c22aebf + id: d2c61505-b860-4812-bb63-ebcb015a226b jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -263,37 +278,44 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: grammatical_between_A_B reference: '' - e629bf12-9c79-4d68-9e26-517c471bd68b: !Template - answer_choices: Yes ||| No - id: e629bf12-9c79-4d68-9e26-517c471bd68b - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + d9523177-55ba-4053-9504-70421382bec8: !Template + answer_choices: 1 ||| 2 + id: d9523177-55ba-4053-9504-70421382bec8 + jinja: '{% set shuffled_order = [0, 1] | random %} - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no - reference: '' - ee810cef-0734-4df6-bb54-9ea41683b943: !Template - answer_choices: Yes ||| No - id: ee810cef-0734-4df6-bb54-9ea41683b943 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + Between the two sentences, which one is grammatical? Please answer 1 or 2. - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: false - name: single_sentence_good_no_yes + original_task: true + name: grammatical_between_1_2 reference: '' diff --git a/promptsource/templates/blimp/sentential_negation_npi_licensor_present/templates.yaml b/promptsource/templates/blimp/sentential_negation_npi_licensor_present/templates.yaml index 6874ae16d..34f989864 100644 --- a/promptsource/templates/blimp/sentential_negation_npi_licensor_present/templates.yaml +++ b/promptsource/templates/blimp/sentential_negation_npi_licensor_present/templates.yaml @@ -1,25 +1,41 @@ dataset: blimp subset: sentential_negation_npi_licensor_present templates: - 15e6a065-a3fb-4a23-a5c7-682853907265: !Template + 036fa9f4-c62a-4bf6-a7fa-e27c1e632653: !Template answer_choices: Yes ||| No - id: 15e6a065-a3fb-4a23-a5c7-682853907265 + id: 036fa9f4-c62a-4bf6-a7fa-e27c1e632653 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_bad_yes_no + name: single_sentence_good_yes_no reference: '' - 1ef568c1-a567-4c97-a284-6478c447a4f9: !Template + 0516d2b3-9634-40b7-a972-91be2d548991: !Template + answer_choices: Yes ||| No + id: 0516d2b3-9634-40b7-a972-91be2d548991 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes + reference: '' + 14a1e0d8-4a24-496d-a559-1d2564cf9286: !Template answer_choices: 1 ||| 2 - id: 1ef568c1-a567-4c97-a284-6478c447a4f9 + id: 14a1e0d8-4a24-496d-a559-1d2564cf9286 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -56,9 +72,9 @@ templates: original_task: true name: grammatical_between_1_2 reference: '' - 21ac6dfd-1678-4b3e-a017-4274c37a1e99: !Template + 333be7e9-ec2d-45b0-87c3-06506dcbc25a: !Template answer_choices: 1 ||| 2 - id: 21ac6dfd-1678-4b3e-a017-4274c37a1e99 + id: 333be7e9-ec2d-45b0-87c3-06506dcbc25a jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer 1 or 2. @@ -95,9 +111,9 @@ templates: original_task: true name: grammatical_which_one_1_2 reference: '' - 44f93830-67ca-4c38-8ec4-9c1d05762e52: !Template + 791a5874-dfc3-4ddf-be8c-7db76caa9a53: !Template answer_choices: A ||| B - id: 44f93830-67ca-4c38-8ec4-9c1d05762e52 + id: 791a5874-dfc3-4ddf-be8c-7db76caa9a53 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -133,30 +149,56 @@ templates: original_task: true name: better_B_A reference: '' - 759dd383-5fa6-4966-ad16-6037ed96cdb6: !Template + b6240669-85c7-4129-aae3-5327cd0a8b3c: !Template + answer_choices: Yes ||| No + id: b6240669-85c7-4129-aae3-5327cd0a8b3c + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes + reference: '' + b90b77dc-ed80-42ce-97d6-6fbd08741c4c: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: b90b77dc-ed80-42ce-97d6-6fbd08741c4c + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: true_null_prompt + reference: '' + ca2e8271-bf69-4575-8ccd-406d8cf3140c: !Template answer_choices: A ||| B - id: 759dd383-5fa6-4966-ad16-6037ed96cdb6 + id: ca2e8271-bf69-4575-8ccd-406d8cf3140c jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -170,14 +212,14 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: better_A_B reference: '' - b1cea338-a852-4e95-b12b-6e34c4102afa: !Template + df82eee5-5fb0-4ec3-ad76-22483ca78977: !Template answer_choices: A ||| B - id: b1cea338-a852-4e95-b12b-6e34c4102afa + id: df82eee5-5fb0-4ec3-ad76-22483ca78977 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -209,63 +251,32 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B - reference: '' - ca401a85-9a0c-4152-b2a7-7a46687acf75: !Template - answer_choices: Yes ||| No - id: ca401a85-9a0c-4152-b2a7-7a46687acf75 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes - reference: '' - de077405-43ec-4b5c-b1e4-8ef8a9b7726e: !Template - answer_choices: Yes ||| No - id: de077405-43ec-4b5c-b1e4-8ef8a9b7726e - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes + name: grammatical_which_one_A_B reference: '' - e9632649-6761-4648-a785-989545485a48: !Template + e10ae3f2-4bee-44fd-9062-0f66b6093055: !Template answer_choices: A ||| B - id: e9632649-6761-4648-a785-989545485a48 + id: e10ae3f2-4bee-44fd-9062-0f66b6093055 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -279,21 +290,32 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: grammatical_between_A_B + reference: '' + e23d2087-8e37-42ed-a2f8-b4fb98f9277c: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: e23d2087-8e37-42ed-a2f8-b4fb98f9277c + jinja: '" ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt reference: '' - ea8d173a-11a8-4755-b9a4-1425c00af1d5: !Template + f138662a-f1d4-49f7-9b61-96af87329a16: !Template answer_choices: Yes ||| No - id: ea8d173a-11a8-4755-b9a4-1425c00af1d5 + id: f138662a-f1d4-49f7-9b61-96af87329a16 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_good_yes_no + name: single_sentence_bad_yes_no reference: '' diff --git a/promptsource/templates/blimp/sentential_negation_npi_scope/templates.yaml b/promptsource/templates/blimp/sentential_negation_npi_scope/templates.yaml index f2833ac3c..ce045818d 100644 --- a/promptsource/templates/blimp/sentential_negation_npi_scope/templates.yaml +++ b/promptsource/templates/blimp/sentential_negation_npi_scope/templates.yaml @@ -1,25 +1,9 @@ dataset: blimp subset: sentential_negation_npi_scope templates: - 63e93f22-a757-4861-8581-5d22d4fedf10: !Template + 21a9ee61-c118-48d8-8230-9562e93b46ab: !Template answer_choices: Yes ||| No - id: 63e93f22-a757-4861-8581-5d22d4fedf10 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes - reference: '' - 6a6def12-011c-47eb-a54b-0ebc1ecdd062: !Template - answer_choices: Yes ||| No - id: 6a6def12-011c-47eb-a54b-0ebc1ecdd062 + id: 21a9ee61-c118-48d8-8230-9562e93b46ab jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -33,30 +17,29 @@ templates: original_task: false name: single_sentence_good_yes_no reference: '' - 6d87f4c2-8e7e-4653-a9fd-e5b2edd0d0da: !Template - answer_choices: 1 ||| 2 - id: 6d87f4c2-8e7e-4653-a9fd-e5b2edd0d0da + 330c84ad-8cb3-43a7-ac74-9b7fbcf47f0e: !Template + answer_choices: A ||| B + id: 330c84ad-8cb3-43a7-ac74-9b7fbcf47f0e jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -70,14 +53,14 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: better_B_A reference: '' - 7691fb0c-048a-49ce-815f-910736e57d2f: !Template + 4e94ed9f-bff1-4859-8b8f-fd6901325b79: !Template answer_choices: 1 ||| 2 - id: 7691fb0c-048a-49ce-815f-910736e57d2f + id: 4e94ed9f-bff1-4859-8b8f-fd6901325b79 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -109,47 +92,32 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 - reference: '' - 9810324c-5934-43eb-a6af-00bda7decb2a: !Template - answer_choices: Yes ||| No - id: 9810324c-5934-43eb-a6af-00bda7decb2a - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no + name: grammatical_which_one_1_2 reference: '' - a27d0b6d-0f6b-4d3f-aa90-4a76eb9d1c93: !Template + 568cc2c8-2d7a-4988-9cc4-65641f742a86: !Template answer_choices: A ||| B - id: a27d0b6d-0f6b-4d3f-aa90-4a76eb9d1c93 + id: 568cc2c8-2d7a-4988-9cc4-65641f742a86 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -163,31 +131,32 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: grammatical_between_A_B reference: '' - ae8dd951-4b8d-4122-b62f-2f541fe0aa63: !Template + 5c224b82-eca5-44c0-b9db-9730dcc69238: !Template answer_choices: A ||| B - id: ae8dd951-4b8d-4122-b62f-2f541fe0aa63 + id: 5c224b82-eca5-44c0-b9db-9730dcc69238 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -201,32 +170,74 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: grammatical_which_one_A_B + reference: '' + 657e2c7d-fe98-4370-83f9-a766eae08819: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 657e2c7d-fe98-4370-83f9-a766eae08819 + jinja: '" ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + 9886e05d-9bf1-40b2-b977-55edd03132ea: !Template + answer_choices: Yes ||| No + id: 9886e05d-9bf1-40b2-b977-55edd03132ea + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes + reference: '' + c70ee875-e604-4166-a5e6-1afb4516dfbb: !Template + answer_choices: Yes ||| No + id: c70ee875-e604-4166-a5e6-1afb4516dfbb + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes reference: '' - d4e8dbdc-91f7-479a-b66c-0a709aa08d7c: !Template + cf464250-783d-4251-b039-b4dd4d93e9c0: !Template answer_choices: A ||| B - id: d4e8dbdc-91f7-479a-b66c-0a709aa08d7c + id: cf464250-783d-4251-b039-b4dd4d93e9c0 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -240,42 +251,37 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: better_A_B reference: '' - efb2fec6-0f4c-454c-88ea-d54326c27d1a: !Template - answer_choices: Yes ||| No - id: efb2fec6-0f4c-454c-88ea-d54326c27d1a - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' + d5d31a73-10d0-437d-a80e-60689045246e: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: d5d31a73-10d0-437d-a80e-60689045246e + jinja: ' ||| {{ sentence_good }}' metadata: !TemplateMetadata - choices_in_prompt: true + choices_in_prompt: false metrics: - Accuracy - original_task: false - name: single_sentence_bad_no_yes + original_task: true + name: true_null_prompt reference: '' - fef4e9a5-b9aa-4cd7-bfaa-9d9bbc4956a1: !Template - answer_choices: A ||| B - id: fef4e9a5-b9aa-4cd7-bfaa-9d9bbc4956a1 + eb95abfa-ed88-49f0-aaaa-518ec91f069e: !Template + answer_choices: 1 ||| 2 + id: eb95abfa-ed88-49f0-aaaa-518ec91f069e jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -295,5 +301,21 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: grammatical_between_1_2 + reference: '' + ed344823-e633-4cfb-a81b-20f7991647a5: !Template + answer_choices: Yes ||| No + id: ed344823-e633-4cfb-a81b-20f7991647a5 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no reference: '' diff --git a/promptsource/templates/blimp/sentential_subject_island/templates.yaml b/promptsource/templates/blimp/sentential_subject_island/templates.yaml index 1671b143f..5fb6caf02 100644 --- a/promptsource/templates/blimp/sentential_subject_island/templates.yaml +++ b/promptsource/templates/blimp/sentential_subject_island/templates.yaml @@ -1,12 +1,62 @@ dataset: blimp subset: sentential_subject_island templates: - 01c97538-2dd1-4950-b762-e980947844b5: !Template + 09965253-1a19-47d0-9da0-1aa906090e23: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 09965253-1a19-47d0-9da0-1aa906090e23 + jinja: '" ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + 26f49f2a-8c9e-44d9-9f88-cd5cbdee4c1b: !Template + answer_choices: 1 ||| 2 + id: 26f49f2a-8c9e-44d9-9f88-cd5cbdee4c1b + jinja: '{% set shuffled_order = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer 1 or 2. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_1_2 + reference: '' + 3e615c5b-16f2-46fd-aafd-3c951c6a8b6f: !Template answer_choices: A ||| B - id: 01c97538-2dd1-4950-b762-e980947844b5 + id: 3e615c5b-16f2-46fd-aafd-3c951c6a8b6f jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -38,14 +88,30 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: grammatical_which_one_A_B reference: '' - 2f70d0e5-6ecb-4dcb-afa4-7dd32665eeaa: !Template + 5fe40b27-b394-4048-9f8c-e52e19bf8531: !Template + answer_choices: Yes ||| No + id: 5fe40b27-b394-4048-9f8c-e52e19bf8531 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 6c18d59c-346a-4244-90da-97cf5183c3c2: !Template answer_choices: 1 ||| 2 - id: 2f70d0e5-6ecb-4dcb-afa4-7dd32665eeaa + id: 6c18d59c-346a-4244-90da-97cf5183c3c2 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -77,11 +143,27 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: grammatical_between_1_2 reference: '' - 43d6b55a-70f3-4336-8c0f-915b44c86a01: !Template + 6c7f8921-fd6d-424a-bad8-0793619eb265: !Template + answer_choices: Yes ||| No + id: 6c7f8921-fd6d-424a-bad8-0793619eb265 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes + reference: '' + 71eb6673-8bc7-421d-9825-1a05270b90d3: !Template answer_choices: A ||| B - id: 43d6b55a-70f3-4336-8c0f-915b44c86a01 + id: 71eb6673-8bc7-421d-9825-1a05270b90d3 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -117,9 +199,9 @@ templates: original_task: true name: better_B_A reference: '' - 715ea764-29c6-4bf5-941e-b7189a6f5fce: !Template + 7f841f3f-5a05-47d2-b1c3-2ffd297eaf09: !Template answer_choices: A ||| B - id: 715ea764-29c6-4bf5-941e-b7189a6f5fce + id: 7f841f3f-5a05-47d2-b1c3-2ffd297eaf09 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -155,52 +237,40 @@ templates: original_task: true name: better_A_B reference: '' - a54d6794-30cf-43b6-930f-e521581243fc: !Template - answer_choices: 1 ||| 2 - id: a54d6794-30cf-43b6-930f-e521581243fc - jinja: '{% set shuffled_order = [0, 1] | random %} - - Between the two sentences, which one is grammatical? Please answer 1 or 2. - - {% if shuffled_order == 0 %} - - 1: {{ sentence_good }} - - 2: {{ sentence_bad }} - - {% else %} - - 1: {{ sentence_bad }} - - 2: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} + 7fe9d8f2-728d-41eb-ada2-32919cb9ca86: !Template + answer_choices: Yes ||| No + id: 7fe9d8f2-728d-41eb-ada2-32919cb9ca86 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - {% endif %}' + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy + original_task: false + name: single_sentence_bad_no_yes + reference: '' + c17ecf33-f485-4c0a-86a4-c672a78f29e8: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: c17ecf33-f485-4c0a-86a4-c672a78f29e8 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy original_task: true - name: grammatical_between_1_2 + name: true_null_prompt reference: '' - ac00abca-5ad0-4498-b2c7-7915205e5207: !Template + c7f6ee4a-2340-4082-96c4-df9cd51dc3de: !Template answer_choices: Yes ||| No - id: ac00abca-5ad0-4498-b2c7-7915205e5207 + id: c7f6ee4a-2340-4082-96c4-df9cd51dc3de jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -208,14 +278,14 @@ templates: metrics: - Accuracy original_task: false - name: single_sentence_good_no_yes + name: single_sentence_good_yes_no reference: '' - bdbe0233-1eaf-47ed-af52-718c16a8a966: !Template + e37d7e7a-bf02-4975-8e0b-a2587b82a62c: !Template answer_choices: A ||| B - id: bdbe0233-1eaf-47ed-af52-718c16a8a966 + id: e37d7e7a-bf02-4975-8e0b-a2587b82a62c jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -247,53 +317,5 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B - reference: '' - bdc0016f-41c3-45ff-b109-fe514ff65a19: !Template - answer_choices: Yes ||| No - id: bdc0016f-41c3-45ff-b109-fe514ff65a19 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no - reference: '' - ebbc3bb5-02cc-4d3a-8a24-44d1faf7cdf7: !Template - answer_choices: Yes ||| No - id: ebbc3bb5-02cc-4d3a-8a24-44d1faf7cdf7 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes - reference: '' - f666c04d-183c-42dc-85f8-992dab34ec3f: !Template - answer_choices: Yes ||| No - id: f666c04d-183c-42dc-85f8-992dab34ec3f - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no + name: grammatical_between_A_B reference: '' diff --git a/promptsource/templates/blimp/superlative_quantifiers_1/templates.yaml b/promptsource/templates/blimp/superlative_quantifiers_1/templates.yaml index 424c2d954..22786a5dc 100644 --- a/promptsource/templates/blimp/superlative_quantifiers_1/templates.yaml +++ b/promptsource/templates/blimp/superlative_quantifiers_1/templates.yaml @@ -1,9 +1,9 @@ dataset: blimp subset: superlative_quantifiers_1 templates: - 16215d7d-0159-4f11-bd52-e44d6ec7627a: !Template + 04d7705a-1887-4442-b1e5-c98417693344: !Template answer_choices: 1 ||| 2 - id: 16215d7d-0159-4f11-bd52-e44d6ec7627a + id: 04d7705a-1887-4442-b1e5-c98417693344 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -40,29 +40,30 @@ templates: original_task: true name: grammatical_between_1_2 reference: '' - 3b474c05-3dd0-4052-a311-c7b93bd00953: !Template + 07c9edae-c9df-4d4e-ad29-cbf68f2748b5: !Template answer_choices: A ||| B - id: 3b474c05-3dd0-4052-a311-c7b93bd00953 + id: 07c9edae-c9df-4d4e-ad29-cbf68f2748b5 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -76,11 +77,11 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: grammatical_between_A_B reference: '' - 42f68a66-02b9-4010-b3da-756cf61553c8: !Template + 0befc6d8-ac12-4a6f-bc96-0a19fece81d9: !Template answer_choices: Yes ||| No - id: 42f68a66-02b9-4010-b3da-756cf61553c8 + id: 0befc6d8-ac12-4a6f-bc96-0a19fece81d9 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -94,9 +95,9 @@ templates: original_task: false name: single_sentence_good_yes_no reference: '' - 48fd0552-beca-4ad7-812b-324960109f33: !Template + 0f5d590d-1668-46a4-a080-a97d148545f0: !Template answer_choices: A ||| B - id: 48fd0552-beca-4ad7-812b-324960109f33 + id: 0f5d590d-1668-46a4-a080-a97d148545f0 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer A or B. @@ -133,9 +134,25 @@ templates: original_task: true name: grammatical_which_one_A_B reference: '' - 5985d362-e21b-4066-8edb-db530a01d25d: !Template + 123d4fa9-8527-4c8b-8418-ded4443a1575: !Template answer_choices: Yes ||| No - id: 5985d362-e21b-4066-8edb-db530a01d25d + id: 123d4fa9-8527-4c8b-8418-ded4443a1575 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes + reference: '' + 1c438fa3-398d-48fe-b41c-4be44bf197cb: !Template + answer_choices: Yes ||| No + id: 1c438fa3-398d-48fe-b41c-4be44bf197cb jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -149,29 +166,57 @@ templates: original_task: false name: single_sentence_good_no_yes reference: '' - abe584d4-28dd-4797-ae93-09aa5c6508d8: !Template - answer_choices: A ||| B - id: abe584d4-28dd-4797-ae93-09aa5c6508d8 + 3d37f387-e142-4e4e-b741-a3d3535d5df8: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 3d37f387-e142-4e4e-b741-a3d3535d5df8 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: true_null_prompt + reference: '' + 452041bb-fcd3-4eb9-96cc-d69465a96c9f: !Template + answer_choices: Yes ||| No + id: 452041bb-fcd3-4eb9-96cc-d69465a96c9f + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 6235e720-b936-4ebf-9c3f-08e958e84015: !Template + answer_choices: 1 ||| 2 + id: 6235e720-b936-4ebf-9c3f-08e958e84015 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -185,32 +230,31 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: grammatical_which_one_1_2 reference: '' - b096d7eb-ac86-4773-bb28-11c18402a04b: !Template - answer_choices: 1 ||| 2 - id: b096d7eb-ac86-4773-bb28-11c18402a04b + 75fe932f-d045-4bd6-815b-ed1b2ca7ebdd: !Template + answer_choices: A ||| B + id: 75fe932f-d045-4bd6-815b-ed1b2ca7ebdd jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -224,64 +268,31 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 - reference: '' - b8aa07e6-d21b-41a7-9443-7b4cace14b58: !Template - answer_choices: Yes ||| No - id: b8aa07e6-d21b-41a7-9443-7b4cace14b58 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no - reference: '' - bc904949-593e-41a1-9b6b-f034ddc14cee: !Template - answer_choices: Yes ||| No - id: bc904949-593e-41a1-9b6b-f034ddc14cee - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes + name: better_A_B reference: '' - e5d7b9ab-5949-40ab-a1b2-438f708e9206: !Template + 7d28c5c3-81fe-4dd9-839d-c2e83966f292: !Template answer_choices: A ||| B - id: e5d7b9ab-5949-40ab-a1b2-438f708e9206 + id: 7d28c5c3-81fe-4dd9-839d-c2e83966f292 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -295,5 +306,16 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: better_B_A + reference: '' + 9abe04b0-aebd-4095-a72a-51d2fa1ae515: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 9abe04b0-aebd-4095-a72a-51d2fa1ae515 + jinja: '" ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt reference: '' diff --git a/promptsource/templates/blimp/superlative_quantifiers_2/templates.yaml b/promptsource/templates/blimp/superlative_quantifiers_2/templates.yaml index ca3b8056e..e50c6336a 100644 --- a/promptsource/templates/blimp/superlative_quantifiers_2/templates.yaml +++ b/promptsource/templates/blimp/superlative_quantifiers_2/templates.yaml @@ -1,72 +1,51 @@ dataset: blimp subset: superlative_quantifiers_2 templates: - 12d1a32b-a052-4afd-9ffa-4d035926bd28: !Template + 07a61dea-60bb-4846-aea2-520dd0177768: !Template answer_choices: Yes ||| No - id: 12d1a32b-a052-4afd-9ffa-4d035926bd28 + id: 07a61dea-60bb-4846-aea2-520dd0177768 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes - reference: '' - 18bf1579-5a8d-429b-8111-6fd2485c4a8a: !Template - answer_choices: Yes ||| No - id: 18bf1579-5a8d-429b-8111-6fd2485c4a8a - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_good_yes_no + name: single_sentence_bad_yes_no reference: '' - 281f71ef-6c15-4a2b-955b-0d29d27e209d: !Template - answer_choices: Yes ||| No - id: 281f71ef-6c15-4a2b-955b-0d29d27e209d - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' + 4390642c-6af5-41b5-a624-3abd21582de1: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 4390642c-6af5-41b5-a624-3abd21582de1 + jinja: ' ||| {{ sentence_good }}' metadata: !TemplateMetadata - choices_in_prompt: true + choices_in_prompt: false metrics: - Accuracy - original_task: false - name: single_sentence_bad_no_yes + original_task: true + name: true_null_prompt reference: '' - 3b6d68ca-ec93-4c48-be59-c70d76395226: !Template - answer_choices: A ||| B - id: 3b6d68ca-ec93-4c48-be59-c70d76395226 + 6161ef6e-2a0b-4f08-84ad-0a91512d588b: !Template + answer_choices: 1 ||| 2 + id: 6161ef6e-2a0b-4f08-84ad-0a91512d588b jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -86,31 +65,32 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: grammatical_between_1_2 reference: '' - 62142866-a88e-409c-a00c-5430bc02a17e: !Template + 79299f1c-b62e-488f-b403-f026155c7e7b: !Template answer_choices: A ||| B - id: 62142866-a88e-409c-a00c-5430bc02a17e + id: 79299f1c-b62e-488f-b403-f026155c7e7b jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -124,11 +104,22 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: grammatical_which_one_A_B reference: '' - 6606bf9f-0678-4feb-b3b5-74496ad49387: !Template + 819a1ab8-aa4c-4d7d-a5d2-ddb2c54cdc86: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 819a1ab8-aa4c-4d7d-a5d2-ddb2c54cdc86 + jinja: '" ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + 874c2efc-10a6-4b5c-9034-e4a00dbdeb4a: !Template answer_choices: A ||| B - id: 6606bf9f-0678-4feb-b3b5-74496ad49387 + id: 874c2efc-10a6-4b5c-9034-e4a00dbdeb4a jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -164,25 +155,9 @@ templates: original_task: true name: better_A_B reference: '' - 68c9bcd3-a7b3-43cb-af83-4356b029cd12: !Template - answer_choices: Yes ||| No - id: 68c9bcd3-a7b3-43cb-af83-4356b029cd12 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no - reference: '' - 775f4a9c-5955-46a7-90b6-9690745b0aa5: !Template + 9c3f75ab-e93c-45f5-8f4c-49c6c124c365: !Template answer_choices: 1 ||| 2 - id: 775f4a9c-5955-46a7-90b6-9690745b0aa5 + id: 9c3f75ab-e93c-45f5-8f4c-49c6c124c365 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer 1 or 2. @@ -219,9 +194,25 @@ templates: original_task: true name: grammatical_which_one_1_2 reference: '' - a6f91c04-05ef-48e7-abbc-4d3aae328e7b: !Template + 9c989643-08c2-462d-9648-9f5232eccd8c: !Template + answer_choices: Yes ||| No + id: 9c989643-08c2-462d-9648-9f5232eccd8c + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes + reference: '' + b3062ae3-0863-4f21-8e39-fa9401c4e72f: !Template answer_choices: A ||| B - id: a6f91c04-05ef-48e7-abbc-4d3aae328e7b + id: b3062ae3-0863-4f21-8e39-fa9401c4e72f jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer A or B. @@ -258,30 +249,45 @@ templates: original_task: true name: grammatical_between_A_B reference: '' - e68f5f96-b39a-4153-a9cb-7f08f5ccf235: !Template - answer_choices: 1 ||| 2 - id: e68f5f96-b39a-4153-a9cb-7f08f5ccf235 + d4b9c969-2d13-484a-b861-d8c7466c1c3b: !Template + answer_choices: Yes ||| No + id: d4b9c969-2d13-484a-b861-d8c7466c1c3b + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes + reference: '' + daf254e1-8a32-479b-a572-1f96ba37875e: !Template + answer_choices: A ||| B + id: daf254e1-8a32-479b-a572-1f96ba37875e jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -295,5 +301,21 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: better_B_A + reference: '' + fafa88ee-fce7-4858-bd61-3510fe2cdabf: !Template + answer_choices: Yes ||| No + id: fafa88ee-fce7-4858-bd61-3510fe2cdabf + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no reference: '' diff --git a/promptsource/templates/blimp/tough_vs_raising_1/templates.yaml b/promptsource/templates/blimp/tough_vs_raising_1/templates.yaml index 73f2e8b0f..392e1f8d7 100644 --- a/promptsource/templates/blimp/tough_vs_raising_1/templates.yaml +++ b/promptsource/templates/blimp/tough_vs_raising_1/templates.yaml @@ -1,40 +1,35 @@ dataset: blimp subset: tough_vs_raising_1 templates: - 14d15e3b-d9c2-4470-a886-b125ca6625d7: !Template - answer_choices: Yes ||| No - id: 14d15e3b-d9c2-4470-a886-b125ca6625d7 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + 08e32861-a173-46ee-9dc2-8a0552d1579c: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 08e32861-a173-46ee-9dc2-8a0552d1579c + jinja: ' ||| {{ sentence_good }}' metadata: !TemplateMetadata - choices_in_prompt: true + choices_in_prompt: false metrics: - Accuracy - original_task: false - name: single_sentence_good_no_yes + original_task: true + name: true_null_prompt reference: '' - 471a0e2a-dcc5-46c3-ba4c-0fdb16801dbf: !Template - answer_choices: A ||| B - id: 471a0e2a-dcc5-46c3-ba4c-0fdb16801dbf + 23644448-b1d3-4c1b-81a5-98fdb5d96c12: !Template + answer_choices: 1 ||| 2 + id: 23644448-b1d3-4c1b-81a5-98fdb5d96c12 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -54,47 +49,48 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: grammatical_which_one_1_2 reference: '' - 7c683c93-8b88-47f8-ab71-5cc6757cb336: !Template + 29f259ac-0258-4427-a777-27c198dd1d06: !Template answer_choices: Yes ||| No - id: 7c683c93-8b88-47f8-ab71-5cc6757cb336 + id: 29f259ac-0258-4427-a777-27c198dd1d06 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_bad_yes_no + name: single_sentence_good_no_yes reference: '' - 920f31d5-b4b9-4d0f-a010-1025b8f960c0: !Template + 3846e1e0-b475-407c-bbad-08c966f66665: !Template answer_choices: A ||| B - id: 920f31d5-b4b9-4d0f-a010-1025b8f960c0 + id: 3846e1e0-b475-407c-bbad-08c966f66665 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -108,30 +104,57 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: grammatical_between_A_B + reference: '' + 3a14af17-ec8f-408e-981a-af88a8712d36: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 3a14af17-ec8f-408e-981a-af88a8712d36 + jinja: '" ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt reference: '' - ae9fb107-6313-419c-a478-ab002e17450b: !Template + 403e805b-74d5-4869-b8c8-ea0a457d79ec: !Template answer_choices: Yes ||| No - id: ae9fb107-6313-419c-a478-ab002e17450b + id: 403e805b-74d5-4869-b8c8-ea0a457d79ec jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_good_yes_no + name: single_sentence_bad_yes_no + reference: '' + 7aa3d847-b2c4-4928-8c57-29ac8e4ef72a: !Template + answer_choices: Yes ||| No + id: 7aa3d847-b2c4-4928-8c57-29ac8e4ef72a + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes reference: '' - d3552793-a5a3-4c64-9630-9e167fcd58f6: !Template + 9b76dce4-2c4f-4245-8a11-42e3a66fd05b: !Template answer_choices: 1 ||| 2 - id: d3552793-a5a3-4c64-9630-9e167fcd58f6 + id: 9b76dce4-2c4f-4245-8a11-42e3a66fd05b jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -163,26 +186,26 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: grammatical_between_1_2 reference: '' - d918a306-025a-4b85-a3f5-ba8777488120: !Template - answer_choices: 1 ||| 2 - id: d918a306-025a-4b85-a3f5-ba8777488120 + c2340ff3-cd1b-45d7-a976-ba64d86eada9: !Template + answer_choices: A ||| B + id: c2340ff3-cd1b-45d7-a976-ba64d86eada9 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -202,11 +225,11 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: grammatical_which_one_A_B reference: '' - de63fc9f-d970-4435-aca7-e4351920bf17: !Template + c2b75382-b113-4a3a-8175-68cccf5fce32: !Template answer_choices: A ||| B - id: de63fc9f-d970-4435-aca7-e4351920bf17 + id: c2b75382-b113-4a3a-8175-68cccf5fce32 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. @@ -225,7 +248,7 @@ templates: {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order + Which one is a better sentence of English, B or A? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -240,48 +263,31 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B - reference: '' - e0ec6447-53e2-48cf-a9de-7b1e539a55f0: !Template - answer_choices: Yes ||| No - id: e0ec6447-53e2-48cf-a9de-7b1e539a55f0 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes + name: better_B_A reference: '' - f4074fdb-d1c5-457b-8a53-f6140338db36: !Template + c4b2a526-4671-4905-9419-837a132de34e: !Template answer_choices: A ||| B - id: f4074fdb-d1c5-457b-8a53-f6140338db36 + id: c4b2a526-4671-4905-9419-837a132de34e jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -295,5 +301,21 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: better_A_B + reference: '' + dd2f5326-543d-42f2-b36f-50b9496ba609: !Template + answer_choices: Yes ||| No + id: dd2f5326-543d-42f2-b36f-50b9496ba609 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no reference: '' diff --git a/promptsource/templates/blimp/tough_vs_raising_2/templates.yaml b/promptsource/templates/blimp/tough_vs_raising_2/templates.yaml index 28c5aecaa..26a442a65 100644 --- a/promptsource/templates/blimp/tough_vs_raising_2/templates.yaml +++ b/promptsource/templates/blimp/tough_vs_raising_2/templates.yaml @@ -1,29 +1,30 @@ dataset: blimp subset: tough_vs_raising_2 templates: - 001c0904-7795-4bcf-bcfe-d26b8b335e7f: !Template + 0a7ada73-c54a-4742-b4ce-b2595cc8e45a: !Template answer_choices: A ||| B - id: 001c0904-7795-4bcf-bcfe-d26b8b335e7f + id: 0a7ada73-c54a-4742-b4ce-b2595cc8e45a jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -37,30 +38,14 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A - reference: '' - 022c1683-8607-4e99-9570-3a9524f30cf6: !Template - answer_choices: Yes ||| No - id: 022c1683-8607-4e99-9570-3a9524f30cf6 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no + name: grammatical_which_one_A_B reference: '' - 5cef6dec-0c90-4d69-bda4-f73a02536190: !Template + 11e9a98d-07bd-4c4a-aaa5-7805a0208dd9: !Template answer_choices: 1 ||| 2 - id: 5cef6dec-0c90-4d69-bda4-f73a02536190 + id: 11e9a98d-07bd-4c4a-aaa5-7805a0208dd9 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -92,49 +77,27 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: grammatical_between_1_2 reference: '' - 60617f99-d83c-474c-92ef-9c7f69b1b56b: !Template - answer_choices: A ||| B - id: 60617f99-d83c-474c-92ef-9c7f69b1b56b - jinja: '{% set shuffled_order = [0, 1] | random %} - - Read the following two sentences. - - {% if shuffled_order == 0 %} - - Sentence A: {{ sentence_good }} - - Sentence B: {{ sentence_bad }} - - {% else %} - - Sentence A: {{ sentence_bad }} - - Sentence B: {{ sentence_good }} - - {% endif %} - - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} + 45afaf9f-f6c7-4644-883c-680ba8b5a742: !Template + answer_choices: Yes ||| No + id: 45afaf9f-f6c7-4644-883c-680ba8b5a742 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - {% endif %}' + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: better_A_B + original_task: false + name: single_sentence_good_yes_no reference: '' - 7d3f7996-76cf-4cb6-ac39-829e697eb9c2: !Template + 6570c687-48d8-4f48-976e-6e6892d00ec1: !Template answer_choices: Yes ||| No - id: 7d3f7996-76cf-4cb6-ac39-829e697eb9c2 + id: 6570c687-48d8-4f48-976e-6e6892d00ec1 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -148,12 +111,39 @@ templates: original_task: false name: single_sentence_bad_no_yes reference: '' - 7d92c27a-05aa-49a8-b37a-a4ecc8e0590a: !Template + 66ba5409-e49f-4259-ab8a-cd0f14a51122: !Template + answer_choices: Yes ||| No + id: 66ba5409-e49f-4259-ab8a-cd0f14a51122 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 6a27f6de-42c5-458d-b0fb-661ae16b37d9: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 6a27f6de-42c5-458d-b0fb-661ae16b37d9 + jinja: '" ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + a572d598-74b4-4f6c-ba1e-57c2961d34e4: !Template answer_choices: A ||| B - id: 7d92c27a-05aa-49a8-b37a-a4ecc8e0590a + id: a572d598-74b4-4f6c-ba1e-57c2961d34e4 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -185,48 +175,31 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B - reference: '' - 90962099-c490-4a7c-9c9c-be0074b2ace5: !Template - answer_choices: Yes ||| No - id: 90962099-c490-4a7c-9c9c-be0074b2ace5 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes + name: grammatical_between_A_B reference: '' - 9909addf-0c61-439a-9886-ac5c9b5fccf2: !Template + a5a408a7-df55-4433-839c-1773a68c5c4e: !Template answer_choices: A ||| B - id: 9909addf-0c61-439a-9886-ac5c9b5fccf2 + id: a5a408a7-df55-4433-839c-1773a68c5c4e jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -240,14 +213,14 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: better_A_B reference: '' - d24ae7a3-9e0e-4f4a-8359-5b28872effc1: !Template + abcac683-f725-493d-8fdb-886a8ec9b173: !Template answer_choices: 1 ||| 2 - id: d24ae7a3-9e0e-4f4a-8359-5b28872effc1 + id: abcac683-f725-493d-8fdb-886a8ec9b173 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -279,15 +252,64 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: grammatical_which_one_1_2 + reference: '' + d3328908-30b5-43f8-b7b6-e9554e9495a0: !Template + answer_choices: A ||| B + id: d3328908-30b5-43f8-b7b6-e9554e9495a0 + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: better_B_A reference: '' - de5df7ee-9d96-41b6-ab45-c277be0847c9: !Template + d515fca1-feda-4cb8-9209-e42eb2319502: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: d515fca1-feda-4cb8-9209-e42eb2319502 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: true_null_prompt + reference: '' + f9905ae2-4060-410d-aa47-e075c4f5434a: !Template answer_choices: Yes ||| No - id: de5df7ee-9d96-41b6-ab45-c277be0847c9 + id: f9905ae2-4060-410d-aa47-e075c4f5434a jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -295,5 +317,5 @@ templates: metrics: - Accuracy original_task: false - name: single_sentence_good_yes_no + name: single_sentence_good_no_yes reference: '' diff --git a/promptsource/templates/blimp/transitive/templates.yaml b/promptsource/templates/blimp/transitive/templates.yaml index e3e73b782..47dfc48cd 100644 --- a/promptsource/templates/blimp/transitive/templates.yaml +++ b/promptsource/templates/blimp/transitive/templates.yaml @@ -1,61 +1,30 @@ dataset: blimp subset: transitive templates: - 2304b1e7-d649-4d97-8806-d29282922ce6: !Template - answer_choices: Yes ||| No - id: 2304b1e7-d649-4d97-8806-d29282922ce6 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no - reference: '' - 43d3db4f-08e1-487f-a8ae-977c38b90a5e: !Template - answer_choices: Yes ||| No - id: 43d3db4f-08e1-487f-a8ae-977c38b90a5e - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no - reference: '' - 4f6d3983-521b-4b51-812e-1ce2532e4ede: !Template - answer_choices: A ||| B - id: 4f6d3983-521b-4b51-812e-1ce2532e4ede + 0eba4768-cb21-40cd-9bdd-db9e6eb2e1a3: !Template + answer_choices: 1 ||| 2 + id: 0eba4768-cb21-40cd-9bdd-db9e6eb2e1a3 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -69,14 +38,14 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: grammatical_which_one_1_2 reference: '' - 65ee533c-1889-421e-810b-7bfd6a80e49a: !Template + 5f50ba61-41f0-4a65-a36a-54870101dc27: !Template answer_choices: A ||| B - id: 65ee533c-1889-421e-810b-7bfd6a80e49a + id: 5f50ba61-41f0-4a65-a36a-54870101dc27 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -108,27 +77,38 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: grammatical_between_A_B + reference: '' + 62dc4b8e-5c20-407a-8f1b-417197081e31: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 62dc4b8e-5c20-407a-8f1b-417197081e31 + jinja: '" ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt reference: '' - 828a65c0-fffc-4daa-98e8-6bee5ddbe725: !Template + 6a1c35fa-4ef4-49bf-a5a2-62055468e338: !Template answer_choices: Yes ||| No - id: 828a65c0-fffc-4daa-98e8-6bee5ddbe725 + id: 6a1c35fa-4ef4-49bf-a5a2-62055468e338 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_good_no_yes + name: single_sentence_bad_yes_no reference: '' - 9c5a2a72-bf6d-463c-a463-576badc80558: !Template + 6b66dc4d-8eeb-4b1b-94eb-fb50bb9502d7: !Template answer_choices: 1 ||| 2 - id: 9c5a2a72-bf6d-463c-a463-576badc80558 + id: 6b66dc4d-8eeb-4b1b-94eb-fb50bb9502d7 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -165,40 +145,40 @@ templates: original_task: true name: grammatical_between_1_2 reference: '' - a8b45ec5-64b2-465e-a371-775fe0e8e8e7: !Template + 6e0ee564-add5-4cac-8217-858f910f8d03: !Template answer_choices: Yes ||| No - id: a8b45ec5-64b2-465e-a371-775fe0e8e8e7 + id: 6e0ee564-add5-4cac-8217-858f910f8d03 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_bad_no_yes + name: single_sentence_good_yes_no reference: '' - aa2d5917-5c3c-453e-ac3f-936ba57a5a2d: !Template - answer_choices: 1 ||| 2 - id: aa2d5917-5c3c-453e-ac3f-936ba57a5a2d + acebb0cb-612c-4cad-a40a-d2c7b5c7cdb6: !Template + answer_choices: A ||| B + id: acebb0cb-612c-4cad-a40a-d2c7b5c7cdb6 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -218,24 +198,51 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: grammatical_which_one_A_B + reference: '' + c15255e9-83fe-4b83-81d3-4894ab7d366f: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: c15255e9-83fe-4b83-81d3-4894ab7d366f + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: true_null_prompt + reference: '' + d2e5a8a9-8b38-400b-8dff-4198560eac9b: !Template + answer_choices: Yes ||| No + id: d2e5a8a9-8b38-400b-8dff-4198560eac9b + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes reference: '' - ae9210af-cfd5-47e0-b321-15397f59ffcd: !Template + d7f89c1a-8780-4c5b-8928-73ae6763d36b: !Template answer_choices: A ||| B - id: ae9210af-cfd5-47e0-b321-15397f59ffcd + id: d7f89c1a-8780-4c5b-8928-73ae6763d36b jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + Sentence A: {{ sentence_good }} Sentence B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} Sentence B: {{ sentence_good }} @@ -258,30 +265,45 @@ templates: original_task: true name: better_A_B reference: '' - d9af92c2-9c54-4d14-a4c0-aa8e48e18dd3: !Template + ea92d1bb-5e82-41ed-bade-dca54a0f6f70: !Template + answer_choices: Yes ||| No + id: ea92d1bb-5e82-41ed-bade-dca54a0f6f70 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes + reference: '' + ff10bbd1-3a44-42af-988d-7e45cc216750: !Template answer_choices: A ||| B - id: d9af92c2-9c54-4d14-a4c0-aa8e48e18dd3 + id: ff10bbd1-3a44-42af-988d-7e45cc216750 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -295,5 +317,5 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: better_B_A reference: '' diff --git a/promptsource/templates/blimp/wh_island/templates.yaml b/promptsource/templates/blimp/wh_island/templates.yaml index 59022755e..88bdf586e 100644 --- a/promptsource/templates/blimp/wh_island/templates.yaml +++ b/promptsource/templates/blimp/wh_island/templates.yaml @@ -1,29 +1,30 @@ dataset: blimp subset: wh_island templates: - 1277e2b2-f41d-4f95-a752-52fd92213676: !Template - answer_choices: A ||| B - id: 1277e2b2-f41d-4f95-a752-52fd92213676 + 2bca2bb6-1bb1-4c51-985a-d85f1c6060c4: !Template + answer_choices: 1 ||| 2 + id: 2bca2bb6-1bb1-4c51-985a-d85f1c6060c4 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -37,31 +38,75 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: grammatical_between_1_2 + reference: '' + 2d3c3858-66c1-45b3-ab88-1c21fd34976f: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 2d3c3858-66c1-45b3-ab88-1c21fd34976f + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: true_null_prompt + reference: '' + 2fd453fc-36de-4441-b4d5-93ba518e3de1: !Template + answer_choices: Yes ||| No + id: 2fd453fc-36de-4441-b4d5-93ba518e3de1 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes + reference: '' + 4adeda8c-a1ce-45bf-8435-2a76c4eb3375: !Template + answer_choices: Yes ||| No + id: 4adeda8c-a1ce-45bf-8435-2a76c4eb3375 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no reference: '' - 196d9fd5-f091-4ea2-afa7-29a6523df0d2: !Template + 54087567-96f7-4573-93ca-d1ca0b3bdee5: !Template answer_choices: A ||| B - id: 196d9fd5-f091-4ea2-afa7-29a6523df0d2 + id: 54087567-96f7-4573-93ca-d1ca0b3bdee5 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -75,32 +120,31 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: grammatical_between_A_B reference: '' - 31d1b252-0b79-4742-aa0a-2303a5b5b43a: !Template + 6508efe4-f064-4f47-84d3-a6d4fa289b18: !Template answer_choices: A ||| B - id: 31d1b252-0b79-4742-aa0a-2303a5b5b43a + id: 6508efe4-f064-4f47-84d3-a6d4fa289b18 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -114,26 +158,26 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: better_B_A reference: '' - 8e4060b6-7786-4714-9cc4-50bb61a5f791: !Template - answer_choices: A ||| B - id: 8e4060b6-7786-4714-9cc4-50bb61a5f791 + 74b49b39-8cd0-4348-a0dc-5e6b46720c43: !Template + answer_choices: 1 ||| 2 + id: 74b49b39-8cd0-4348-a0dc-5e6b46720c43 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -153,42 +197,42 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: grammatical_which_one_1_2 reference: '' - 941969d9-b0e5-4eea-9295-b2286c18b3ed: !Template + 86c69941-3c5c-444d-a753-3d988d7c5a24: !Template answer_choices: Yes ||| No - id: 941969d9-b0e5-4eea-9295-b2286c18b3ed + id: 86c69941-3c5c-444d-a753-3d988d7c5a24 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_good_no_yes + name: single_sentence_bad_no_yes reference: '' - a4ac23ce-b4c3-4202-a519-6931868eb504: !Template - answer_choices: 1 ||| 2 - id: a4ac23ce-b4c3-4202-a519-6931868eb504 + 8799761e-cfb5-4054-93be-0c13781120fb: !Template + answer_choices: A ||| B + id: 8799761e-cfb5-4054-93be-0c13781120fb jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -208,32 +252,42 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: grammatical_which_one_A_B reference: '' - bbc60b2c-f5e3-4c95-9b5a-04f67e373064: !Template - answer_choices: 1 ||| 2 - id: bbc60b2c-f5e3-4c95-9b5a-04f67e373064 + 8dd56f62-03c2-4187-b09a-f259c1c27aa2: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 8dd56f62-03c2-4187-b09a-f259c1c27aa2 + jinja: '" ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + e3100602-a400-41b6-ab3f-2e8c0cac93ee: !Template + answer_choices: A ||| B + id: e3100602-a400-41b6-ab3f-2e8c0cac93ee jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -247,43 +301,11 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 - reference: '' - befe5d14-82e7-4795-ad81-96109373982f: !Template - answer_choices: Yes ||| No - id: befe5d14-82e7-4795-ad81-96109373982f - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no - reference: '' - c8f0b100-c673-4703-afb2-0aa47511360c: !Template - answer_choices: Yes ||| No - id: c8f0b100-c673-4703-afb2-0aa47511360c - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes + name: better_A_B reference: '' - e4a84272-1b18-4db0-8d22-eafb4032f5e5: !Template + ea6793cc-cb1b-4fd0-a79b-c649991add1c: !Template answer_choices: Yes ||| No - id: e4a84272-1b18-4db0-8d22-eafb4032f5e5 + id: ea6793cc-cb1b-4fd0-a79b-c649991add1c jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" diff --git a/promptsource/templates/blimp/wh_questions_object_gap/templates.yaml b/promptsource/templates/blimp/wh_questions_object_gap/templates.yaml index 071ed4d45..e41eab632 100644 --- a/promptsource/templates/blimp/wh_questions_object_gap/templates.yaml +++ b/promptsource/templates/blimp/wh_questions_object_gap/templates.yaml @@ -1,24 +1,40 @@ dataset: blimp subset: wh_questions_object_gap templates: - 0ba81e3c-e360-48b7-98fd-2ef7955c1607: !Template - answer_choices: A ||| B - id: 0ba81e3c-e360-48b7-98fd-2ef7955c1607 + 1275a82f-a19d-4fed-858d-2eb27c6472ba: !Template + answer_choices: Yes ||| No + id: 1275a82f-a19d-4fed-858d-2eb27c6472ba + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 296af7e5-b024-47cd-b52d-98b8abce3ce9: !Template + answer_choices: 1 ||| 2 + id: 296af7e5-b024-47cd-b52d-98b8abce3ce9 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -38,48 +54,31 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B - reference: '' - 16dcde34-9933-464a-8927-7556d034db0f: !Template - answer_choices: Yes ||| No - id: 16dcde34-9933-464a-8927-7556d034db0f - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes + name: grammatical_which_one_1_2 reference: '' - 3bbe15f8-6ded-4aa1-beea-6adfb9bbf323: !Template + 2c46a0f3-ccee-486d-9153-2d1409ba3061: !Template answer_choices: A ||| B - id: 3bbe15f8-6ded-4aa1-beea-6adfb9bbf323 + id: 2c46a0f3-ccee-486d-9153-2d1409ba3061 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -93,11 +92,38 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: better_B_A reference: '' - 670e3dae-14cd-46a9-a356-77ad86e317b1: !Template + 4f7c3aee-750b-4569-8d2f-a35d8b240f96: !Template + answer_choices: Yes ||| No + id: 4f7c3aee-750b-4569-8d2f-a35d8b240f96 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes + reference: '' + 68365a2c-b632-4905-98c1-6fefe801103b: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 68365a2c-b632-4905-98c1-6fefe801103b + jinja: '" ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + 6bd5fce7-9e2c-4f82-bfe6-9229f1738260: !Template answer_choices: 1 ||| 2 - id: 670e3dae-14cd-46a9-a356-77ad86e317b1 + id: 6bd5fce7-9e2c-4f82-bfe6-9229f1738260 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -134,29 +160,46 @@ templates: original_task: true name: grammatical_between_1_2 reference: '' - 76324e7f-cda6-4e89-ad8e-817fbe507b44: !Template + 7925636e-0693-42ab-825e-35bdc44350c9: !Template + answer_choices: Yes ||| No + id: 7925636e-0693-42ab-825e-35bdc44350c9 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 83a017c4-405f-40b7-9b33-bc2219c4136b: !Template answer_choices: A ||| B - id: 76324e7f-cda6-4e89-ad8e-817fbe507b44 + id: 83a017c4-405f-40b7-9b33-bc2219c4136b jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -170,47 +213,15 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A - reference: '' - 87415348-9aa3-4cdb-9600-0e48d5e18ee8: !Template - answer_choices: Yes ||| No - id: 87415348-9aa3-4cdb-9600-0e48d5e18ee8 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes - reference: '' - 91f5de4d-2392-4230-9cfb-41237333f246: !Template - answer_choices: Yes ||| No - id: 91f5de4d-2392-4230-9cfb-41237333f246 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no + name: grammatical_which_one_A_B reference: '' - ad9ce087-f787-46aa-936e-08f09a2b3b6b: !Template + 8c61d74b-b230-42c4-8cf6-60a92288eaeb: !Template answer_choices: Yes ||| No - id: ad9ce087-f787-46aa-936e-08f09a2b3b6b + id: 8c61d74b-b230-42c4-8cf6-60a92288eaeb jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -218,31 +229,32 @@ templates: metrics: - Accuracy original_task: false - name: single_sentence_good_yes_no + name: single_sentence_good_no_yes reference: '' - b8da5dfa-5eaa-4140-a864-a288371a71ff: !Template + b76f0d57-1881-447b-88f9-44b1fadf8399: !Template answer_choices: A ||| B - id: b8da5dfa-5eaa-4140-a864-a288371a71ff + id: b76f0d57-1881-447b-88f9-44b1fadf8399 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -256,32 +268,42 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: grammatical_between_A_B reference: '' - ce574003-2e89-4e0c-a36b-85c8ae02560a: !Template - answer_choices: 1 ||| 2 - id: ce574003-2e89-4e0c-a36b-85c8ae02560a + c9c0261f-97c4-4cec-941c-b241a09bf67c: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: c9c0261f-97c4-4cec-941c-b241a09bf67c + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: true_null_prompt + reference: '' + d65a937f-64f8-4353-b166-0b0540905e55: !Template + answer_choices: A ||| B + id: d65a937f-64f8-4353-b166-0b0540905e55 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -295,5 +317,5 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: better_A_B reference: '' diff --git a/promptsource/templates/blimp/wh_questions_subject_gap/templates.yaml b/promptsource/templates/blimp/wh_questions_subject_gap/templates.yaml index 119de5aa3..8dbf066f6 100644 --- a/promptsource/templates/blimp/wh_questions_subject_gap/templates.yaml +++ b/promptsource/templates/blimp/wh_questions_subject_gap/templates.yaml @@ -1,45 +1,30 @@ dataset: blimp subset: wh_questions_subject_gap templates: - 3be2fbd3-dc8a-42a8-b75f-2bf5f97a9a06: !Template - answer_choices: Yes ||| No - id: 3be2fbd3-dc8a-42a8-b75f-2bf5f97a9a06 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes - reference: '' - 3e74107b-4b89-4a8e-8e87-5b58ea940035: !Template + 0126b33c-f9a6-4bd0-81df-b6da0634a05b: !Template answer_choices: A ||| B - id: 3e74107b-4b89-4a8e-8e87-5b58ea940035 + id: 0126b33c-f9a6-4bd0-81df-b6da0634a05b jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -53,26 +38,26 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: grammatical_which_one_A_B reference: '' - 517be3cd-115a-4310-bf56-53c41e1544da: !Template - answer_choices: A ||| B - id: 517be3cd-115a-4310-bf56-53c41e1544da + 16570efc-68bf-4157-b82c-dab803929cea: !Template + answer_choices: 1 ||| 2 + id: 16570efc-68bf-4157-b82c-dab803929cea jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -92,30 +77,57 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: grammatical_which_one_1_2 reference: '' - 728f9769-e4aa-40e0-bce0-7570cb622caa: !Template + 39547bc9-35e7-40d2-9bae-fe3b91677ef9: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 39547bc9-35e7-40d2-9bae-fe3b91677ef9 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: true_null_prompt + reference: '' + 474feff6-8c2f-40af-9a50-07304f248440: !Template + answer_choices: Yes ||| No + id: 474feff6-8c2f-40af-9a50-07304f248440 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 4c021b8e-1135-4dcc-b11d-ce18fab28f02: !Template answer_choices: A ||| B - id: 728f9769-e4aa-40e0-bce0-7570cb622caa + id: 4c021b8e-1135-4dcc-b11d-ce18fab28f02 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + Sentence A: {{ sentence_good }} Sentence B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} Sentence B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order + Which one is a better sentence of English, A or B? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -130,26 +142,26 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: better_A_B reference: '' - aef524e8-5ff7-4dd0-bcab-015fb33a2818: !Template - answer_choices: A ||| B - id: aef524e8-5ff7-4dd0-bcab-015fb33a2818 + 5190dcc5-c105-4c4e-831f-fe6b8fe3ad1f: !Template + answer_choices: 1 ||| 2 + id: 5190dcc5-c105-4c4e-831f-fe6b8fe3ad1f jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -169,58 +181,26 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B - reference: '' - b4f8537b-64f3-48ff-b445-3d9cf3e1b285: !Template - answer_choices: Yes ||| No - id: b4f8537b-64f3-48ff-b445-3d9cf3e1b285 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no - reference: '' - c39e11fa-c69f-4fb1-b24a-ac154ddc0667: !Template - answer_choices: Yes ||| No - id: c39e11fa-c69f-4fb1-b24a-ac154ddc0667 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes + name: grammatical_between_1_2 reference: '' - c494a50d-45e9-4ece-a9fb-869930283074: !Template - answer_choices: 1 ||| 2 - id: c494a50d-45e9-4ece-a9fb-869930283074 + a419a20a-e200-4c3f-822b-3293807164a1: !Template + answer_choices: A ||| B + id: a419a20a-e200-4c3f-822b-3293807164a1 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -240,11 +220,11 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: grammatical_between_A_B reference: '' - c8b8a68e-d91e-4db8-9698-4c6aa15db7e6: !Template + a6af6beb-c9d9-42c0-8ac8-20a980f97328: !Template answer_choices: Yes ||| No - id: c8b8a68e-d91e-4db8-9698-4c6aa15db7e6 + id: a6af6beb-c9d9-42c0-8ac8-20a980f97328 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -258,30 +238,72 @@ templates: original_task: false name: single_sentence_good_yes_no reference: '' - ddf494f1-47ae-4c4a-a726-8b9ff4bfff04: !Template - answer_choices: 1 ||| 2 - id: ddf494f1-47ae-4c4a-a726-8b9ff4bfff04 + af764f44-c852-4a97-bcd2-efb0dc8d9fac: !Template + answer_choices: Yes ||| No + id: af764f44-c852-4a97-bcd2-efb0dc8d9fac + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes + reference: '' + b57011b1-bdaa-4274-a2ce-be4a9bcd62f5: !Template + answer_choices: Yes ||| No + id: b57011b1-bdaa-4274-a2ce-be4a9bcd62f5 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes + reference: '' + eb6f0983-bf35-4d1c-9f32-17a986fc1374: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: eb6f0983-bf35-4d1c-9f32-17a986fc1374 + jinja: '" ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + ef016243-c2eb-423f-a533-fcc60f3655fb: !Template + answer_choices: A ||| B + id: ef016243-c2eb-423f-a533-fcc60f3655fb jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -295,5 +317,5 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: better_B_A reference: '' diff --git a/promptsource/templates/blimp/wh_questions_subject_gap_long_distance/templates.yaml b/promptsource/templates/blimp/wh_questions_subject_gap_long_distance/templates.yaml index 8f028bbb5..7099161f3 100644 --- a/promptsource/templates/blimp/wh_questions_subject_gap_long_distance/templates.yaml +++ b/promptsource/templates/blimp/wh_questions_subject_gap_long_distance/templates.yaml @@ -1,62 +1,29 @@ dataset: blimp subset: wh_questions_subject_gap_long_distance templates: - 28ebe131-6988-4bb0-a157-a0f7accef5c8: !Template - answer_choices: Yes ||| No - id: 28ebe131-6988-4bb0-a157-a0f7accef5c8 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no - reference: '' - 4eaf3a66-94d5-4e78-a0e0-fbf73ac9fccd: !Template - answer_choices: Yes ||| No - id: 4eaf3a66-94d5-4e78-a0e0-fbf73ac9fccd - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes - reference: '' - 4eba01cc-3f89-4ee2-9847-b4078b71a224: !Template - answer_choices: 1 ||| 2 - id: 4eba01cc-3f89-4ee2-9847-b4078b71a224 + 00d475f6-2219-4a2f-9b59-0af9790af10d: !Template + answer_choices: A ||| B + id: 00d475f6-2219-4a2f-9b59-0af9790af10d jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -70,26 +37,26 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: better_B_A reference: '' - 5b04b4a2-11ad-4976-8e7c-dae6f15651a8: !Template - answer_choices: 1 ||| 2 - id: 5b04b4a2-11ad-4976-8e7c-dae6f15651a8 + 4073e6da-b998-4523-af4a-2662455efba8: !Template + answer_choices: A ||| B + id: 4073e6da-b998-4523-af4a-2662455efba8 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -109,26 +76,42 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: grammatical_which_one_A_B reference: '' - 8ce39c09-68b5-4356-9a5d-50f698776bdf: !Template - answer_choices: A ||| B - id: 8ce39c09-68b5-4356-9a5d-50f698776bdf + 6555c3e9-e408-475b-a828-e0fb864c0aa9: !Template + answer_choices: Yes ||| No + id: 6555c3e9-e408-475b-a828-e0fb864c0aa9 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 7484d736-9848-46b3-903a-5d603ee81987: !Template + answer_choices: 1 ||| 2 + id: 7484d736-9848-46b3-903a-5d603ee81987 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -148,31 +131,32 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: grammatical_which_one_1_2 reference: '' - b6f677a8-d25b-4c30-a818-9e7abe2edca8: !Template - answer_choices: A ||| B - id: b6f677a8-d25b-4c30-a818-9e7abe2edca8 + 85d4c1ca-20bf-47e0-89d3-4289d05357d8: !Template + answer_choices: 1 ||| 2 + id: 85d4c1ca-20bf-47e0-89d3-4289d05357d8 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -186,27 +170,22 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: grammatical_between_1_2 reference: '' - c7607b5c-2383-41de-8e05-5a3975d5be2d: !Template - answer_choices: Yes ||| No - id: c7607b5c-2383-41de-8e05-5a3975d5be2d - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' + 89d1c1ea-568e-4b48-9771-2396af2f3f1a: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 89d1c1ea-568e-4b48-9771-2396af2f3f1a + jinja: ' ||| {{ sentence_good }}' metadata: !TemplateMetadata - choices_in_prompt: true + choices_in_prompt: false metrics: - Accuracy - original_task: false - name: single_sentence_bad_no_yes + original_task: true + name: true_null_prompt reference: '' - d385fbf9-6bbb-4d9f-96e3-b3e039d6679d: !Template + ad11e240-e8ea-4a4a-8413-a30589831045: !Template answer_choices: A ||| B - id: d385fbf9-6bbb-4d9f-96e3-b3e039d6679d + id: ad11e240-e8ea-4a4a-8413-a30589831045 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer A or B. @@ -243,22 +222,22 @@ templates: original_task: true name: grammatical_between_A_B reference: '' - ecab1d1c-7f5a-488b-b2a9-f8d233c1d832: !Template + b33871dc-c6f9-4531-b09e-6ffb72d7bc21: !Template answer_choices: A ||| B - id: ecab1d1c-7f5a-488b-b2a9-f8d233c1d832 + id: b33871dc-c6f9-4531-b09e-6ffb72d7bc21 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + Sentence A: {{ sentence_good }} Sentence B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} Sentence B: {{ sentence_good }} @@ -281,9 +260,52 @@ templates: original_task: true name: better_A_B reference: '' - f477c93b-9cd6-4504-8dc4-0e9e871a8c8c: !Template + ca1ba3be-8c69-4d67-ba03-6ddb9fd148b3: !Template + answer_choices: Yes ||| No + id: ca1ba3be-8c69-4d67-ba03-6ddb9fd148b3 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes + reference: '' + dc90181a-2866-42ce-864b-7330ec839234: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: dc90181a-2866-42ce-864b-7330ec839234 + jinja: '" ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + e985f87a-6a81-4df7-b705-fa090af87c21: !Template + answer_choices: Yes ||| No + id: e985f87a-6a81-4df7-b705-fa090af87c21 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes + reference: '' + fe81c28a-0587-4f3f-8613-081b2cee2efe: !Template answer_choices: Yes ||| No - id: f477c93b-9cd6-4504-8dc4-0e9e871a8c8c + id: fe81c28a-0587-4f3f-8613-081b2cee2efe jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" diff --git a/promptsource/templates/blimp/wh_vs_that_no_gap/templates.yaml b/promptsource/templates/blimp/wh_vs_that_no_gap/templates.yaml index b891cd408..9397055c9 100644 --- a/promptsource/templates/blimp/wh_vs_that_no_gap/templates.yaml +++ b/promptsource/templates/blimp/wh_vs_that_no_gap/templates.yaml @@ -1,25 +1,58 @@ dataset: blimp subset: wh_vs_that_no_gap templates: - 02fe381d-25e7-42c9-8cba-05ef1c299ea5: !Template - answer_choices: Yes ||| No - id: 02fe381d-25e7-42c9-8cba-05ef1c299ea5 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + 03861afc-89b4-4b90-9d5a-167c38df6cf2: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 03861afc-89b4-4b90-9d5a-167c38df6cf2 + jinja: '" ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + 1a950cbd-dea6-41be-ab64-29034549bbf9: !Template + answer_choices: A ||| B + id: 1a950cbd-dea6-41be-ab64-29034549bbf9 + jinja: '{% set shuffled_order = [0, 1] | random %} - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: false - name: single_sentence_good_yes_no + original_task: true + name: better_B_A reference: '' - 1c9d0f71-728e-470c-940e-20eca46eb36e: !Template + 1f0b1a4b-20c6-4217-8d64-8acfc1413b6d: !Template answer_choices: A ||| B - id: 1c9d0f71-728e-470c-940e-20eca46eb36e + id: 1f0b1a4b-20c6-4217-8d64-8acfc1413b6d jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer A or B. @@ -56,9 +89,41 @@ templates: original_task: true name: grammatical_between_A_B reference: '' - 2a5149f4-35e0-471b-92e5-41a8f6c12e6b: !Template + 5d0c1b60-c6cb-480d-8c82-c1bacc3eaec8: !Template + answer_choices: Yes ||| No + id: 5d0c1b60-c6cb-480d-8c82-c1bacc3eaec8 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 5eb1161b-df0b-4ff4-b4c8-c29456ed2481: !Template + answer_choices: Yes ||| No + id: 5eb1161b-df0b-4ff4-b4c8-c29456ed2481 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes + reference: '' + 64d71314-cde2-45d9-a782-7cf28a502a62: !Template answer_choices: 1 ||| 2 - id: 2a5149f4-35e0-471b-92e5-41a8f6c12e6b + id: 64d71314-cde2-45d9-a782-7cf28a502a62 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -95,29 +160,30 @@ templates: original_task: true name: grammatical_between_1_2 reference: '' - 4f35487d-2f76-41ca-88cd-ec1771800dc7: !Template - answer_choices: A ||| B - id: 4f35487d-2f76-41ca-88cd-ec1771800dc7 + 7221e9ee-e560-4b98-b263-7042c51b84a1: !Template + answer_choices: 1 ||| 2 + id: 7221e9ee-e560-4b98-b263-7042c51b84a1 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -131,46 +197,30 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B - reference: '' - 6c800564-f349-4328-a858-9982c8373a11: !Template - answer_choices: Yes ||| No - id: 6c800564-f349-4328-a858-9982c8373a11 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no + name: grammatical_which_one_1_2 reference: '' - 723c2dcf-9f76-401e-b0b7-d6c19db0900f: !Template + b00be7cf-766c-4268-97eb-b04ecb10b565: !Template answer_choices: A ||| B - id: 723c2dcf-9f76-401e-b0b7-d6c19db0900f + id: b00be7cf-766c-4268-97eb-b04ecb10b565 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + Sentence A: {{ sentence_good }} Sentence B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} Sentence B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order + Which one is a better sentence of English, A or B? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -185,11 +235,11 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: better_A_B reference: '' - 932625dd-b96e-4d24-bdff-abcf59dbeead: !Template + ba0cf74f-a1dc-4183-a573-4b6ea71b7bd0: !Template answer_choices: A ||| B - id: 932625dd-b96e-4d24-bdff-abcf59dbeead + id: ba0cf74f-a1dc-4183-a573-4b6ea71b7bd0 jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer A or B. @@ -226,9 +276,9 @@ templates: original_task: true name: grammatical_which_one_A_B reference: '' - bd7515c0-8c86-4e76-b456-bd4a0b9b9e46: !Template + c298b934-e17a-4175-96d8-998e7669af42: !Template answer_choices: Yes ||| No - id: bd7515c0-8c86-4e76-b456-bd4a0b9b9e46 + id: c298b934-e17a-4175-96d8-998e7669af42 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" @@ -242,13 +292,24 @@ templates: original_task: false name: single_sentence_bad_no_yes reference: '' - da4987d5-8a06-4e72-8381-ef17f3c21133: !Template + e5315a89-f463-4fc9-94f3-04d59e96e36c: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: e5315a89-f463-4fc9-94f3-04d59e96e36c + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: true_null_prompt + reference: '' + e8b23765-cc3d-4be1-a5fd-782fc6eeb195: !Template answer_choices: Yes ||| No - id: da4987d5-8a06-4e72-8381-ef17f3c21133 + id: e8b23765-cc3d-4be1-a5fd-782fc6eeb195 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -256,44 +317,5 @@ templates: metrics: - Accuracy original_task: false - name: single_sentence_good_no_yes - reference: '' - e5c16d1f-d8eb-4f79-b027-a450f9776ed4: !Template - answer_choices: 1 ||| 2 - id: e5c16d1f-d8eb-4f79-b027-a450f9776ed4 - jinja: '{% set shuffled_order = [0, 1] | random %} - - Which one of the following sentences is grammatical? Please answer 1 or 2. - - {% if shuffled_order == 0 %} - - 1: {{ sentence_good }} - - 2: {{ sentence_bad }} - - {% else %} - - 1: {{ sentence_bad }} - - 2: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} - - {% endif %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: grammatical_which_one_1_2 + name: single_sentence_good_yes_no reference: '' diff --git a/promptsource/templates/blimp/wh_vs_that_no_gap_long_distance/templates.yaml b/promptsource/templates/blimp/wh_vs_that_no_gap_long_distance/templates.yaml index a4f7e685e..584999e19 100644 --- a/promptsource/templates/blimp/wh_vs_that_no_gap_long_distance/templates.yaml +++ b/promptsource/templates/blimp/wh_vs_that_no_gap_long_distance/templates.yaml @@ -1,51 +1,13 @@ dataset: blimp subset: wh_vs_that_no_gap_long_distance templates: - 090af836-a282-47a4-82b8-1b33708cdc55: !Template - answer_choices: A ||| B - id: 090af836-a282-47a4-82b8-1b33708cdc55 - jinja: '{% set shuffled_order = [0, 1] | random %} - - Read the following two sentences. - - {% if shuffled_order == 0 %} - - Sentence A: {{ sentence_good }} - - Sentence B: {{ sentence_bad }} - - {% else %} - - Sentence A: {{ sentence_bad }} - - Sentence B: {{ sentence_good }} - - {% endif %} - - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} - - {% endif %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: better_B_A - reference: '' - 2fffb3ed-4be4-4bac-a847-0296482de5f1: !Template + 0354119f-5e9f-4787-a5ea-9411712261ec: !Template answer_choices: Yes ||| No - id: 2fffb3ed-4be4-4bac-a847-0296482de5f1 + id: 0354119f-5e9f-4787-a5ea-9411712261ec jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata @@ -53,31 +15,15 @@ templates: metrics: - Accuracy original_task: false - name: single_sentence_bad_yes_no - reference: '' - 5105a5c0-e61c-49ba-b8ff-24b5260c51f9: !Template - answer_choices: Yes ||| No - id: 5105a5c0-e61c-49ba-b8ff-24b5260c51f9 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no + name: single_sentence_bad_no_yes reference: '' - 59bf12ae-b890-42aa-8c15-8bd483732eed: !Template + 1611776d-6dfc-4296-9ce5-ce72b0f4d9ed: !Template answer_choices: Yes ||| No - id: 59bf12ae-b890-42aa-8c15-8bd483732eed + id: 1611776d-6dfc-4296-9ce5-ce72b0f4d9ed jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata @@ -85,30 +31,53 @@ templates: metrics: - Accuracy original_task: false - name: single_sentence_bad_no_yes + name: single_sentence_bad_yes_no reference: '' - 612e1ab2-7ec8-4f52-8e2e-6f3490aff846: !Template - answer_choices: Yes ||| No - id: 612e1ab2-7ec8-4f52-8e2e-6f3490aff846 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + 17fb8eac-e5e6-49c6-8b24-cd67a2472051: !Template + answer_choices: A ||| B + id: 17fb8eac-e5e6-49c6-8b24-cd67a2472051 + jinja: '{% set shuffled_order = [0, 1] | random %} - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + Between the two sentences, which one is grammatical? Please answer A or B. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: false - name: single_sentence_good_no_yes + original_task: true + name: grammatical_between_A_B reference: '' - 6a8803b9-b898-423c-80f6-9dcf587edca7: !Template + 1924cce9-dc96-4daf-9084-e548b1ae265c: !Template answer_choices: 1 ||| 2 - id: 6a8803b9-b898-423c-80f6-9dcf587edca7 + id: 1924cce9-dc96-4daf-9084-e548b1ae265c jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} @@ -140,32 +109,31 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: grammatical_which_one_1_2 reference: '' - a69a6d39-2b8b-4ae4-b7e6-a7e4408be701: !Template + 2b037c6f-7957-4362-a7c8-c2bae9552e6d: !Template answer_choices: A ||| B - id: a69a6d39-2b8b-4ae4-b7e6-a7e4408be701 + id: 2b037c6f-7957-4362-a7c8-c2bae9552e6d jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -179,31 +147,48 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: better_B_A reference: '' - af64e812-5ab7-4d8f-ab82-7f5f8fdf20db: !Template - answer_choices: A ||| B - id: af64e812-5ab7-4d8f-ab82-7f5f8fdf20db + 2f8466d6-5eb9-46d1-9fa1-99e80f589c6f: !Template + answer_choices: Yes ||| No + id: 2f8466d6-5eb9-46d1-9fa1-99e80f589c6f + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_no_yes + reference: '' + 49ff211c-81f7-49e9-8c6d-e47bf62059c9: !Template + answer_choices: 1 ||| 2 + id: 49ff211c-81f7-49e9-8c6d-e47bf62059c9 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -217,32 +202,58 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: grammatical_between_1_2 reference: '' - b5b8e78f-9ef8-4bf2-b4c3-d14fa424ead1: !Template + 54b70e37-2e72-40c4-90f7-9d8bfc767ba3: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 54b70e37-2e72-40c4-90f7-9d8bfc767ba3 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: true_null_prompt + reference: '' + 54f00e83-11ed-403f-9dda-6c4284948004: !Template + answer_choices: Yes ||| No + id: 54f00e83-11ed-403f-9dda-6c4284948004 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 7889bb75-004c-46f0-b4fd-5c32ea98cb21: !Template answer_choices: A ||| B - id: b5b8e78f-9ef8-4bf2-b4c3-d14fa424ead1 + id: 7889bb75-004c-46f0-b4fd-5c32ea98cb21 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -256,26 +267,37 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: better_A_B reference: '' - ff19dd12-be84-4286-b666-1a00a145ac80: !Template - answer_choices: 1 ||| 2 - id: ff19dd12-be84-4286-b666-1a00a145ac80 + be848bb1-ffa6-4478-afb8-93bb284ad415: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: be848bb1-ffa6-4478-afb8-93bb284ad415 + jinja: '" ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + fec5d43c-00ef-4b95-b0d1-a20c22ba6921: !Template + answer_choices: A ||| B + id: fec5d43c-00ef-4b95-b0d1-a20c22ba6921 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -295,5 +317,5 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: grammatical_which_one_A_B reference: '' diff --git a/promptsource/templates/blimp/wh_vs_that_with_gap/templates.yaml b/promptsource/templates/blimp/wh_vs_that_with_gap/templates.yaml index da5423ce2..b273160b2 100644 --- a/promptsource/templates/blimp/wh_vs_that_with_gap/templates.yaml +++ b/promptsource/templates/blimp/wh_vs_that_with_gap/templates.yaml @@ -1,46 +1,40 @@ dataset: blimp subset: wh_vs_that_with_gap templates: - 1a2a0afd-ac1a-4969-bc21-7f2e1dde361a: !Template - answer_choices: Yes ||| No - id: 1a2a0afd-ac1a-4969-bc21-7f2e1dde361a - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' + 06dbc186-11d3-4eec-bb9e-b229dbb87b20: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 06dbc186-11d3-4eec-bb9e-b229dbb87b20 + jinja: '" ||| {{ sentence_good }}' metadata: !TemplateMetadata - choices_in_prompt: true + choices_in_prompt: false metrics: - Accuracy - original_task: false - name: single_sentence_bad_no_yes + original_task: true + name: null_prompt reference: '' - 345bd841-7af7-4381-9a44-ab9497e5c7ce: !Template - answer_choices: 1 ||| 2 - id: 345bd841-7af7-4381-9a44-ab9497e5c7ce + 180ff240-8910-45b6-9992-7987729ec1df: !Template + answer_choices: A ||| B + id: 180ff240-8910-45b6-9992-7987729ec1df jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -54,32 +48,31 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: better_B_A reference: '' - 37019fb3-2423-4f6e-b6c9-12ef27ea0395: !Template + 1b583b7d-5c3f-47dc-bda4-615a73ae111c: !Template answer_choices: A ||| B - id: 37019fb3-2423-4f6e-b6c9-12ef27ea0395 + id: 1b583b7d-5c3f-47dc-bda4-615a73ae111c jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, A or B? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} @@ -93,47 +86,32 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B - reference: '' - 4147d3d1-b692-434f-8d06-f6ca82edb8f3: !Template - answer_choices: Yes ||| No - id: 4147d3d1-b692-434f-8d06-f6ca82edb8f3 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no + name: better_A_B reference: '' - 63005ef4-90e4-4d0e-bd82-54e82eb9118a: !Template - answer_choices: A ||| B - id: 63005ef4-90e4-4d0e-bd82-54e82eb9118a + 48d300fd-82d7-409d-918a-e5bd39bf3c52: !Template + answer_choices: 1 ||| 2 + id: 48d300fd-82d7-409d-918a-e5bd39bf3c52 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -147,11 +125,27 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: grammatical_between_1_2 + reference: '' + 5ef7db69-d3a6-4279-864a-4941c7585954: !Template + answer_choices: Yes ||| No + id: 5ef7db69-d3a6-4279-864a-4941c7585954 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no reference: '' - 77255ece-e099-42db-a582-75477f76f5f7: !Template + 6354bb80-ac81-40e4-838b-8619040863ac: !Template answer_choices: 1 ||| 2 - id: 77255ece-e099-42db-a582-75477f76f5f7 + id: 6354bb80-ac81-40e4-838b-8619040863ac jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer 1 or 2. @@ -188,45 +182,46 @@ templates: original_task: true name: grammatical_which_one_1_2 reference: '' - e92eb612-9467-4be7-aeee-98b8b1772895: !Template + 64fd2d61-2190-4ad1-a891-b21cea26a62e: !Template answer_choices: Yes ||| No - id: e92eb612-9467-4be7-aeee-98b8b1772895 + id: 64fd2d61-2190-4ad1-a891-b21cea26a62e jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_good_no_yes + name: single_sentence_bad_yes_no reference: '' - ecb09615-c2a0-4292-8693-e5ed817e230e: !Template + 8a0f9f77-8cc7-4f43-bb71-ad1a7510f745: !Template answer_choices: A ||| B - id: ecb09615-c2a0-4292-8693-e5ed817e230e + id: 8a0f9f77-8cc7-4f43-bb71-ad1a7510f745 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -240,27 +235,27 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: grammatical_between_A_B reference: '' - ecdeeef8-1672-4dc4-8963-eefb74dc2a5e: !Template + 9f6df466-cf62-4cdb-9d6a-973e2e1192d6: !Template answer_choices: Yes ||| No - id: ecdeeef8-1672-4dc4-8963-eefb74dc2a5e + id: 9f6df466-cf62-4cdb-9d6a-973e2e1192d6 jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_bad_yes_no + name: single_sentence_good_no_yes reference: '' - f53a2291-d705-401a-b4c6-1f02b3fb960c: !Template + a5b3813b-5adf-4a25-a632-700aa4b692cf: !Template answer_choices: A ||| B - id: f53a2291-d705-401a-b4c6-1f02b3fb960c + id: a5b3813b-5adf-4a25-a632-700aa4b692cf jinja: '{% set shuffled_order = [0, 1] | random %} Which one of the following sentences is grammatical? Please answer A or B. @@ -297,3 +292,30 @@ templates: original_task: true name: grammatical_which_one_A_B reference: '' + b86d7436-eff6-4205-8b1b-c634ba749f65: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: b86d7436-eff6-4205-8b1b-c634ba749f65 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: true_null_prompt + reference: '' + fe36c816-722d-4652-9235-04e9101f42f6: !Template + answer_choices: Yes ||| No + id: fe36c816-722d-4652-9235-04e9101f42f6 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_no_yes + reference: '' diff --git a/promptsource/templates/blimp/wh_vs_that_with_gap_long_distance/templates.yaml b/promptsource/templates/blimp/wh_vs_that_with_gap_long_distance/templates.yaml index 5d6162245..712eae388 100644 --- a/promptsource/templates/blimp/wh_vs_that_with_gap_long_distance/templates.yaml +++ b/promptsource/templates/blimp/wh_vs_that_with_gap_long_distance/templates.yaml @@ -1,9 +1,63 @@ dataset: blimp subset: wh_vs_that_with_gap_long_distance templates: - 0b9eb74c-c82e-4709-84b0-85dc9bf6eace: !Template + 05d153b5-35ea-48a9-bf49-2469d840767b: !Template + answer_choices: A ||| B + id: 05d153b5-35ea-48a9-bf49-2469d840767b + jinja: '{% set shuffled_order = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, B or A? ||| {% if shuffled_order + == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: better_B_A + reference: '' + 1a6b7b9b-9b68-48a3-8192-23cc92d0f6a3: !Template + answer_choices: Yes ||| No + id: 1a6b7b9b-9b68-48a3-8192-23cc92d0f6a3 + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 253cc4c1-bfad-49e7-b9cb-fe9b1e82cbb3: !Template answer_choices: 1 ||| 2 - id: 0b9eb74c-c82e-4709-84b0-85dc9bf6eace + id: 253cc4c1-bfad-49e7-b9cb-fe9b1e82cbb3 jinja: '{% set shuffled_order = [0, 1] | random %} Between the two sentences, which one is grammatical? Please answer 1 or 2. @@ -40,38 +94,54 @@ templates: original_task: true name: grammatical_between_1_2 reference: '' - 0e3396fc-9559-4e9d-8ee4-254d55f11710: !Template + 274111ea-7de1-4e69-9298-7c21760d311b: !Template answer_choices: Yes ||| No - id: 0e3396fc-9559-4e9d-8ee4-254d55f11710 + id: 274111ea-7de1-4e69-9298-7c21760d311b jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif + Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_good_no_yes + name: single_sentence_bad_no_yes reference: '' - 1edac537-702e-4086-8ce0-fc0723ab34e2: !Template + 38988ff7-1854-4887-9b02-059bcc36199a: !Template + answer_choices: Yes ||| No + id: 38988ff7-1854-4887-9b02-059bcc36199a + jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ + sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + + Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order + == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 4200922a-2bc2-4992-8408-2de5fdfcf435: !Template answer_choices: A ||| B - id: 1edac537-702e-4086-8ce0-fc0723ab34e2 + id: 4200922a-2bc2-4992-8408-2de5fdfcf435 jinja: '{% set shuffled_order = [0, 1] | random %} Read the following two sentences. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + Sentence A: {{ sentence_good }} Sentence B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} Sentence B: {{ sentence_good }} @@ -94,12 +164,23 @@ templates: original_task: true name: better_A_B reference: '' - 32853a41-c3fe-4a80-bcaf-eb6be1e7ebc9: !Template + 53101768-0016-4cfb-80a7-81581fcf7adc: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 53101768-0016-4cfb-80a7-81581fcf7adc + jinja: '" ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + 58644c43-ed51-4f3c-9f96-bea0bfe2892c: !Template answer_choices: A ||| B - id: 32853a41-c3fe-4a80-bcaf-eb6be1e7ebc9 + id: 58644c43-ed51-4f3c-9f96-bea0bfe2892c jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + Which one of the following sentences is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -131,31 +212,32 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: grammatical_which_one_A_B reference: '' - 458ea68a-0659-40dc-ac9a-478cb828cc1f: !Template - answer_choices: A ||| B - id: 458ea68a-0659-40dc-ac9a-478cb828cc1f + 5ea2aede-1e64-434d-a019-d9f969ca38ed: !Template + answer_choices: 1 ||| 2 + id: 5ea2aede-1e64-434d-a019-d9f969ca38ed jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + Which one of the following sentences is grammatical? Please answer 1 or 2. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -169,31 +251,26 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: grammatical_which_one_1_2 reference: '' - bb086356-5183-46c8-aab7-dc28e814c35d: !Template - answer_choices: Yes ||| No - id: bb086356-5183-46c8-aab7-dc28e814c35d - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' + 879299dd-fc1f-41cc-93b1-df2a7a141f8a: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 879299dd-fc1f-41cc-93b1-df2a7a141f8a + jinja: ' ||| {{ sentence_good }}' metadata: !TemplateMetadata - choices_in_prompt: true + choices_in_prompt: false metrics: - Accuracy - original_task: false - name: single_sentence_bad_yes_no + original_task: true + name: true_null_prompt reference: '' - bc533a50-f176-4c73-8cc7-2ab0819bc641: !Template + c6f3615b-a641-4d8d-b104-d11a31b7da7e: !Template answer_choices: Yes ||| No - id: bc533a50-f176-4c73-8cc7-2ab0819bc641 + id: c6f3615b-a641-4d8d-b104-d11a31b7da7e jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order + Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata @@ -201,30 +278,14 @@ templates: metrics: - Accuracy original_task: false - name: single_sentence_good_yes_no - reference: '' - c9f796cb-e35d-4128-8d2a-5fdd6c86f24f: !Template - answer_choices: Yes ||| No - id: c9f796cb-e35d-4128-8d2a-5fdd6c86f24f - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes + name: single_sentence_good_no_yes reference: '' - d36bdeb7-6316-4dd5-a029-9db2ecd3ef11: !Template + ebc93391-c378-471e-b821-2e944a555018: !Template answer_choices: A ||| B - id: d36bdeb7-6316-4dd5-a029-9db2ecd3ef11 + id: ebc93391-c378-471e-b821-2e944a555018 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + Between the two sentences, which one is grammatical? Please answer A or B. {% if shuffled_order == 0 %} @@ -256,44 +317,5 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B - reference: '' - eff28ba0-08ec-4149-bbe6-c3a9a6fab62a: !Template - answer_choices: 1 ||| 2 - id: eff28ba0-08ec-4149-bbe6-c3a9a6fab62a - jinja: '{% set shuffled_order = [0, 1] | random %} - - Which one of the following sentences is grammatical? Please answer 1 or 2. - - {% if shuffled_order == 0 %} - - 1: {{ sentence_good }} - - 2: {{ sentence_bad }} - - {% else %} - - 1: {{ sentence_bad }} - - 2: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} - - {% endif %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: grammatical_which_one_1_2 + name: grammatical_between_A_B reference: '' From a0a451d43eefbd968aec70284f6b347223227833 Mon Sep 17 00:00:00 2001 From: najoungkim Date: Wed, 27 Apr 2022 16:47:54 -0400 Subject: [PATCH 15/16] Minor cleanup --- promptsource/templates.py | 1 - 1 file changed, 1 deletion(-) diff --git a/promptsource/templates.py b/promptsource/templates.py index 02353c203..3ece927fe 100644 --- a/promptsource/templates.py +++ b/promptsource/templates.py @@ -380,7 +380,6 @@ def read_from_file(self) -> Dict: """ if not os.path.exists(self.yaml_path): - print('============yaml path============', self.yaml_path) dataset_name = f"{self.dataset_name} {self.subset_name}" if self.subset_name else self.dataset_name logging.warning( f"Tried instantiating `DatasetTemplates` for {dataset_name}, but no prompts found. " From 7a37755a74f1d82a2fd13ce3a446a5f5e8d5f7a9 Mon Sep 17 00:00:00 2001 From: najoungkim Date: Wed, 27 Apr 2022 19:05:22 -0400 Subject: [PATCH 16/16] Prompt cleanups: true null prompt, choice randomization + dropping order swapped prompts. --- .../blimp/adjunct_island/templates.yaml | 137 +++------ .../anaphor_gender_agreement/templates.yaml | 251 +++++++--------- .../anaphor_number_agreement/templates.yaml | 217 +++++--------- .../animate_subject_passive/templates.yaml | 213 +++++--------- .../animate_subject_trans/templates.yaml | 213 +++++--------- .../templates/blimp/causative/templates.yaml | 219 ++++++-------- .../blimp/complex_NP_island/templates.yaml | 239 ++++++---------- .../templates.yaml | 233 ++++++--------- .../templates.yaml | 259 +++++++---------- .../templates.yaml | 225 ++++++--------- .../templates.yaml | 237 ++++++--------- .../templates.yaml | 239 ++++++---------- .../templates.yaml | 215 +++++--------- .../templates.yaml | 251 +++++++--------- .../templates.yaml | 223 ++++++--------- .../templates.yaml | 227 ++++++--------- .../templates.yaml | 231 ++++++--------- .../templates.yaml | 231 ++++++--------- .../templates.yaml | 211 +++++--------- .../blimp/drop_argument/templates.yaml | 231 ++++++--------- .../blimp/ellipsis_n_bar_1/templates.yaml | 207 +++++--------- .../blimp/ellipsis_n_bar_2/templates.yaml | 251 +++++++--------- .../templates.yaml | 215 +++++--------- .../templates.yaml | 269 +++++++----------- .../templates.yaml | 251 +++++++--------- .../templates.yaml | 241 ++++++---------- .../templates.yaml | 227 ++++++--------- .../templates/blimp/inchoative/templates.yaml | 237 ++++++--------- .../blimp/intransitive/templates.yaml | 247 ++++++---------- .../templates.yaml | 239 ++++++---------- .../templates.yaml | 267 +++++++---------- .../templates.yaml | 217 +++++--------- .../templates.yaml | 189 +++++------- .../templates.yaml | 223 ++++++--------- .../templates.yaml | 189 +++++------- .../templates.yaml | 225 ++++++--------- .../blimp/npi_present_1/templates.yaml | 223 ++++++--------- .../blimp/npi_present_2/templates.yaml | 245 ++++++---------- .../only_npi_licensor_present/templates.yaml | 235 ++++++--------- .../blimp/only_npi_scope/templates.yaml | 249 +++++++--------- .../templates/blimp/passive_1/templates.yaml | 225 ++++++--------- .../templates/blimp/passive_2/templates.yaml | 231 ++++++--------- .../principle_A_c_command/templates.yaml | 237 ++++++--------- .../blimp/principle_A_case_1/templates.yaml | 241 ++++++---------- .../blimp/principle_A_case_2/templates.yaml | 213 +++++--------- .../blimp/principle_A_domain_1/templates.yaml | 225 ++++++--------- .../blimp/principle_A_domain_2/templates.yaml | 245 ++++++---------- .../blimp/principle_A_domain_3/templates.yaml | 255 +++++++---------- .../principle_A_reconstruction/templates.yaml | 203 +++++-------- .../templates.yaml | 241 ++++++---------- .../templates.yaml | 253 +++++++--------- .../templates.yaml | 257 +++++++---------- .../templates.yaml | 215 +++++--------- .../sentential_subject_island/templates.yaml | 231 ++++++--------- .../superlative_quantifiers_1/templates.yaml | 201 +++++-------- .../superlative_quantifiers_2/templates.yaml | 245 ++++++---------- .../blimp/tough_vs_raising_1/templates.yaml | 195 +++++-------- .../blimp/tough_vs_raising_2/templates.yaml | 211 +++++--------- .../templates/blimp/transitive/templates.yaml | 237 ++++++--------- .../templates/blimp/wh_island/templates.yaml | 223 ++++++--------- .../wh_questions_object_gap/templates.yaml | 251 +++++++--------- .../wh_questions_subject_gap/templates.yaml | 251 +++++++--------- .../templates.yaml | 213 +++++--------- .../blimp/wh_vs_that_no_gap/templates.yaml | 243 ++++++---------- .../templates.yaml | 227 ++++++--------- .../blimp/wh_vs_that_with_gap/templates.yaml | 245 ++++++---------- .../templates.yaml | 227 ++++++--------- 67 files changed, 5718 insertions(+), 9671 deletions(-) diff --git a/promptsource/templates/blimp/adjunct_island/templates.yaml b/promptsource/templates/blimp/adjunct_island/templates.yaml index ff863c633..7181abf50 100644 --- a/promptsource/templates/blimp/adjunct_island/templates.yaml +++ b/promptsource/templates/blimp/adjunct_island/templates.yaml @@ -1,50 +1,15 @@ dataset: blimp subset: adjunct_island templates: - 3d2821a7-1c9c-44dd-ad23-e964b0998d1b: !Template - answer_choices: A ||| B - id: 3d2821a7-1c9c-44dd-ad23-e964b0998d1b - jinja: '{% set shuffled_order = [0, 1] | random %} - - Read the following two sentences. - - {% if shuffled_order == 0 %} - - Sentence A: {{ sentence_good }} - - Sentence B: {{ sentence_bad }} - - {% else %} - - Sentence A: {{ sentence_bad }} - - Sentence B: {{ sentence_good }} - - {% endif %} - - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} - - {% endif %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: better_B_A - reference: '' 6e71cd4b-882c-4458-9cd4-fd83f849ad09: !Template answer_choices: 1 ||| 2 id: 6e71cd4b-882c-4458-9cd4-fd83f849ad09 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} @@ -78,23 +43,15 @@ templates: original_task: true name: grammatical_between_1_2 reference: '' - 77a180c8-9529-4675-88ea-3b50cb9a093d: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 77a180c8-9529-4675-88ea-3b50cb9a093d - jinja: '" ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: null_prompt - reference: '' 7ed733fd-524c-4f00-800f-77a7a862ab4a: !Template answer_choices: 1 ||| 2 id: 7ed733fd-524c-4f00-800f-77a7a862ab4a jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} @@ -128,31 +85,19 @@ templates: original_task: true name: grammatical_which_one_1_2 reference: '' - 8bb3b001-cda4-43a3-8692-f191393bb2ef: !Template - answer_choices: Yes ||| No - id: 8bb3b001-cda4-43a3-8692-f191393bb2ef - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes - reference: '' 8f354d74-5ffb-4cbc-b119-b79f2db00df3: !Template answer_choices: Yes ||| No id: 8f354d74-5ffb-4cbc-b119-b79f2db00df3 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + jinja: '{% set shuffled_order = [0, 1] | random %} - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: @@ -165,7 +110,10 @@ templates: id: 967ca4e2-162f-4365-a1f5-2c87c0e874ce jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} @@ -202,12 +150,16 @@ templates: a1997343-0a20-4f8c-8939-a512f0f68485: !Template answer_choices: Yes ||| No id: a1997343-0a20-4f8c-8939-a512f0f68485 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: @@ -220,6 +172,8 @@ templates: id: c45d5bac-968e-4e39-96b7-d18a5ddb225c jinja: '{% set shuffled_order = [0, 1] | random %} + {% set shuffled_order_options = [0, 1] | random %} + Read the following two sentences. {% if shuffled_order == 0 %} @@ -236,8 +190,8 @@ templates: {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -258,7 +212,10 @@ templates: id: d11a37d7-7c42-4ee0-96c5-ffd24ad7073e jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} @@ -292,22 +249,6 @@ templates: original_task: true name: grammatical_which_one_A_B reference: '' - f4f94eda-a7ac-485a-82d1-e36012771762: !Template - answer_choices: Yes ||| No - id: f4f94eda-a7ac-485a-82d1-e36012771762 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes - reference: '' f6cfe60f-a0c5-4ab8-ae7d-1c08d3cdaa2b: !Template answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' id: f6cfe60f-a0c5-4ab8-ae7d-1c08d3cdaa2b @@ -317,5 +258,5 @@ templates: metrics: - Accuracy original_task: true - name: true_null_prompt + name: null_prompt reference: '' diff --git a/promptsource/templates/blimp/anaphor_gender_agreement/templates.yaml b/promptsource/templates/blimp/anaphor_gender_agreement/templates.yaml index b26c0b22a..73a8bd566 100644 --- a/promptsource/templates/blimp/anaphor_gender_agreement/templates.yaml +++ b/promptsource/templates/blimp/anaphor_gender_agreement/templates.yaml @@ -1,68 +1,73 @@ dataset: blimp subset: anaphor_gender_agreement templates: - 28c2b5aa-29ce-4d36-aa80-daf33f3f5c57: !Template - answer_choices: 1 ||| 2 - id: 28c2b5aa-29ce-4d36-aa80-daf33f3f5c57 + 07c38f60-08de-4ab9-9d85-10a6dc19d604: !Template + answer_choices: Yes ||| No + id: 07c38f60-08de-4ab9-9d85-10a6dc19d604 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. - - {% if shuffled_order == 0 %} - - 1: {{ sentence_good }} - - 2: {{ sentence_bad }} - - {% else %} + {% set shuffled_order_options = [0, 1] | random %} - 1: {{ sentence_bad }} - - 2: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" - {{ answer_choices[0] }} + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 0bd4cee1-1ad7-491e-a035-da04db56a34a: !Template + answer_choices: Yes ||| No + id: 0bd4cee1-1ad7-491e-a035-da04db56a34a + jinja: '{% set shuffled_order = [0, 1] | random %} - {% else %} + {% set shuffled_order_options = [0, 1] | random %} - {{ answer_choices[1] }} + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" - {% endif %}' + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: grammatical_which_one_1_2 + original_task: false + name: single_sentence_bad_yes_no reference: '' - 2bc4df5e-6432-494f-a88a-9a22e1c0ab6b: !Template - answer_choices: A ||| B - id: 2bc4df5e-6432-494f-a88a-9a22e1c0ab6b + 4fc6563f-e07e-414c-836b-42ac0ad1bda1: !Template + answer_choices: 1 ||| 2 + id: 4fc6563f-e07e-414c-836b-42ac0ad1bda1 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -76,47 +81,46 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: grammatical_which_one_1_2 reference: '' - 3188a3e3-5cb9-4f03-9087-ef289c16ec68: !Template - answer_choices: Yes ||| No - id: 3188a3e3-5cb9-4f03-9087-ef289c16ec68 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + 715c8035-7ea5-4083-9be3-d58da070a14f: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 715c8035-7ea5-4083-9be3-d58da070a14f + jinja: ' ||| {{ sentence_good }}' metadata: !TemplateMetadata - choices_in_prompt: true + choices_in_prompt: false metrics: - Accuracy - original_task: false - name: single_sentence_good_no_yes + original_task: true + name: null_prompt reference: '' - 4b3a2ad2-494f-4a3f-b296-9fe185edc1d8: !Template - answer_choices: A ||| B - id: 4b3a2ad2-494f-4a3f-b296-9fe185edc1d8 + 77f3fbbd-88d3-432d-81fa-ed6efe7ed1ba: !Template + answer_choices: 1 ||| 2 + id: 77f3fbbd-88d3-432d-81fa-ed6efe7ed1ba jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -130,14 +134,17 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: grammatical_between_1_2 reference: '' - 6c675909-285a-4ed8-89bb-da2e40b288ee: !Template + d78bee9a-0745-4da9-a958-4a169a9c1da2: !Template answer_choices: A ||| B - id: 6c675909-285a-4ed8-89bb-da2e40b288ee + id: d78bee9a-0745-4da9-a958-4a169a9c1da2 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} @@ -169,53 +176,29 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B - reference: '' - 849851e4-f1f9-4c01-b25d-abe588438454: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 849851e4-f1f9-4c01-b25d-abe588438454 - jinja: ' ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: true_null_prompt - reference: '' - 8fd53d64-b95b-45cb-87fc-c9e765bc8b70: !Template - answer_choices: Yes ||| No - id: 8fd53d64-b95b-45cb-87fc-c9e765bc8b70 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no + name: grammatical_between_A_B reference: '' - b50e7d9c-e69b-4827-8333-96a7f5f76bd1: !Template - answer_choices: 1 ||| 2 - id: b50e7d9c-e69b-4827-8333-96a7f5f76bd1 + f83533a9-239e-4534-90e6-51e6f2fcd581: !Template + answer_choices: A ||| B + id: f83533a9-239e-4534-90e6-51e6f2fcd581 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -235,48 +218,33 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 - reference: '' - ca10bff1-9a6c-4281-90b2-23d65916afed: !Template - answer_choices: Yes ||| No - id: ca10bff1-9a6c-4281-90b2-23d65916afed - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes + name: grammatical_which_one_A_B reference: '' - d2eacc09-0036-4dda-adeb-905d891ad306: !Template + fa337eec-293f-49ef-a6ab-19657333eef8: !Template answer_choices: A ||| B - id: d2eacc09-0036-4dda-adeb-905d891ad306 + id: fa337eec-293f-49ef-a6ab-19657333eef8 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -290,32 +258,5 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B - reference: '' - e98d6ef5-acc7-4b5e-b064-a0834f9b084b: !Template - answer_choices: Yes ||| No - id: e98d6ef5-acc7-4b5e-b064-a0834f9b084b - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no - reference: '' - eb865335-7712-4775-b3ea-76c61e7f15d5: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: eb865335-7712-4775-b3ea-76c61e7f15d5 - jinja: '" ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: null_prompt + name: better_A_B reference: '' diff --git a/promptsource/templates/blimp/anaphor_number_agreement/templates.yaml b/promptsource/templates/blimp/anaphor_number_agreement/templates.yaml index a8e644105..177f5739f 100644 --- a/promptsource/templates/blimp/anaphor_number_agreement/templates.yaml +++ b/promptsource/templates/blimp/anaphor_number_agreement/templates.yaml @@ -1,77 +1,44 @@ dataset: blimp subset: anaphor_number_agreement templates: - 0847e8b0-ff2b-4a5f-bd0b-f071c7c3688b: !Template - answer_choices: A ||| B - id: 0847e8b0-ff2b-4a5f-bd0b-f071c7c3688b - jinja: '{% set shuffled_order = [0, 1] | random %} - - Between the two sentences, which one is grammatical? Please answer A or B. - - {% if shuffled_order == 0 %} - - A: {{ sentence_good }} - - B: {{ sentence_bad }} - - {% else %} - - A: {{ sentence_bad }} - - B: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} - - {% endif %}' + 12e8c8b3-23b4-4134-87aa-35c16dcf1b28: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 12e8c8b3-23b4-4134-87aa-35c16dcf1b28 + jinja: ' ||| {{ sentence_good }}' metadata: !TemplateMetadata - choices_in_prompt: true + choices_in_prompt: false metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: null_prompt reference: '' - 637b6e3b-d0fc-4090-8fb6-7ae52709bcdc: !Template + 37795465-6f54-4b97-8801-33fe6963d204: !Template answer_choices: Yes ||| No - id: 637b6e3b-d0fc-4090-8fb6-7ae52709bcdc - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + id: 37795465-6f54-4b97-8801-33fe6963d204 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_bad_yes_no - reference: '' - 6af9b651-b6f4-4fa3-8c8e-2dc942f7d249: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 6af9b651-b6f4-4fa3-8c8e-2dc942f7d249 - jinja: '" ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: null_prompt + name: single_sentence_good_yes_no reference: '' - 6afa5333-ba67-4c5d-a67f-bd00c3cf8a52: !Template + 4b3a2b2c-945e-43a9-b1ed-f80e7a66989c: !Template answer_choices: A ||| B - id: 6afa5333-ba67-4c5d-a67f-bd00c3cf8a52 + id: 4b3a2b2c-945e-43a9-b1ed-f80e7a66989c jinja: '{% set shuffled_order = [0, 1] | random %} + {% set shuffled_order_options = [0, 1] | random %} + Read the following two sentences. {% if shuffled_order == 0 %} @@ -88,8 +55,8 @@ templates: {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -103,57 +70,17 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A - reference: '' - 8e19ca7b-f2ca-4aa9-bb91-363ab48690c8: !Template - answer_choices: Yes ||| No - id: 8e19ca7b-f2ca-4aa9-bb91-363ab48690c8 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes - reference: '' - a654a3ba-a74e-4fd8-90ed-baba061f170b: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: a654a3ba-a74e-4fd8-90ed-baba061f170b - jinja: ' ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: true_null_prompt - reference: '' - a6679425-6cd8-42e7-a7dd-1564a07b457c: !Template - answer_choices: Yes ||| No - id: a6679425-6cd8-42e7-a7dd-1564a07b457c - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes + name: better_A_B reference: '' - ab0505c1-af35-44af-a874-7a579576109a: !Template + 530a3b44-1c87-450e-8964-b349443ceb0c: !Template answer_choices: 1 ||| 2 - id: ab0505c1-af35-44af-a874-7a579576109a + id: 530a3b44-1c87-450e-8964-b349443ceb0c jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} @@ -185,14 +112,17 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: grammatical_between_1_2 reference: '' - b3efa878-3d7c-45bb-9a30-b6c5025a2945: !Template + 80ebb026-8d6d-45c6-aad9-40d21601576e: !Template answer_choices: 1 ||| 2 - id: b3efa878-3d7c-45bb-9a30-b6c5025a2945 + id: 80ebb026-8d6d-45c6-aad9-40d21601576e jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} @@ -224,14 +154,17 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: grammatical_which_one_1_2 reference: '' - e39925fa-2aea-4ec9-81c5-b4a3137eef05: !Template + 813e7a45-e5d1-4a68-9af6-91c6086c125c: !Template answer_choices: A ||| B - id: e39925fa-2aea-4ec9-81c5-b4a3137eef05 + id: 813e7a45-e5d1-4a68-9af6-91c6086c125c jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} @@ -265,45 +198,33 @@ templates: original_task: true name: grammatical_which_one_A_B reference: '' - e5f07808-fdd3-4bcd-a6ed-5c7af1d670de: !Template - answer_choices: Yes ||| No - id: e5f07808-fdd3-4bcd-a6ed-5c7af1d670de - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no - reference: '' - f2edd012-b456-4fbb-9a61-035b16262057: !Template + a4da3db8-8075-4e34-bc38-53a0ac5dde9d: !Template answer_choices: A ||| B - id: f2edd012-b456-4fbb-9a61-035b16262057 + id: a4da3db8-8075-4e34-bc38-53a0ac5dde9d jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -317,5 +238,25 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: grammatical_between_A_B + reference: '' + e96b76bb-923c-4d9d-aa29-2a6ce4f5eb95: !Template + answer_choices: Yes ||| No + id: e96b76bb-923c-4d9d-aa29-2a6ce4f5eb95 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no reference: '' diff --git a/promptsource/templates/blimp/animate_subject_passive/templates.yaml b/promptsource/templates/blimp/animate_subject_passive/templates.yaml index 4aea1fb90..16f9e8529 100644 --- a/promptsource/templates/blimp/animate_subject_passive/templates.yaml +++ b/promptsource/templates/blimp/animate_subject_passive/templates.yaml @@ -1,56 +1,47 @@ dataset: blimp subset: animate_subject_passive templates: - 37db01ee-31a2-46d3-8f05-2b3d19a410d6: !Template + 2bce041a-2ef3-402f-9492-2b203e947897: !Template answer_choices: Yes ||| No - id: 37db01ee-31a2-46d3-8f05-2b3d19a410d6 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + id: 2bce041a-2ef3-402f-9492-2b203e947897 + jinja: '{% set shuffled_order = [0, 1] | random %} - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no - reference: '' - 3fbf0301-1963-4846-8515-8c8169eda731: !Template - answer_choices: Yes ||| No - id: 3fbf0301-1963-4846-8515-8c8169eda731 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_good_no_yes + name: single_sentence_good_yes_no reference: '' - 40ccc24a-7616-45c6-b20a-f34a7a5e5952: !Template - answer_choices: 1 ||| 2 - id: 40ccc24a-7616-45c6-b20a-f34a7a5e5952 + 4e3231f6-f507-4f84-9017-2bf9852ad25e: !Template + answer_choices: A ||| B + id: 4e3231f6-f507-4f84-9017-2bf9852ad25e jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -70,91 +61,49 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: grammatical_which_one_A_B reference: '' - 5780ce16-31b2-4f46-8869-b30649be4a29: !Template + 58ec546c-51f6-461b-9e3a-4eb816be86de: !Template answer_choices: Yes ||| No - id: 5780ce16-31b2-4f46-8869-b30649be4a29 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no - reference: '' - 6721356a-6e94-43f1-8433-7629cc3d8c34: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 6721356a-6e94-43f1-8433-7629cc3d8c34 - jinja: '" ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: null_prompt - reference: '' - 725bb88d-9eb5-493f-9114-c0cbef6418ad: !Template - answer_choices: A ||| B - id: 725bb88d-9eb5-493f-9114-c0cbef6418ad + id: 58ec546c-51f6-461b-9e3a-4eb816be86de jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. - - {% if shuffled_order == 0 %} - - Sentence A: {{ sentence_good }} - - Sentence B: {{ sentence_bad }} - - {% else %} + {% set shuffled_order_options = [0, 1] | random %} - Sentence A: {{ sentence_bad }} - - Sentence B: {{ sentence_good }} + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" - {% endif %} - - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} - - {% endif %}' + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: better_A_B + original_task: false + name: single_sentence_bad_yes_no reference: '' - a6928a46-a55f-4190-b24b-d6f8c40dd5af: !Template - answer_choices: A ||| B - id: a6928a46-a55f-4190-b24b-d6f8c40dd5af + 61f671f5-9308-4627-b560-7c33587ae33b: !Template + answer_choices: 1 ||| 2 + id: 61f671f5-9308-4627-b560-7c33587ae33b jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -174,14 +123,17 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: grammatical_which_one_1_2 reference: '' - c0b7e59d-d702-4e53-9de2-a4e1e8916634: !Template + 76fe5d87-47e1-40f2-896e-8ea3be79a613: !Template answer_choices: 1 ||| 2 - id: c0b7e59d-d702-4e53-9de2-a4e1e8916634 + id: 76fe5d87-47e1-40f2-896e-8ea3be79a613 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} @@ -213,25 +165,17 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 - reference: '' - d673a952-18e4-48f2-a64e-6cad36affa62: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: d673a952-18e4-48f2-a64e-6cad36affa62 - jinja: ' ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: true_null_prompt + name: grammatical_between_1_2 reference: '' - d6f53c48-9d17-440f-bc3d-6d01005d2ab4: !Template + 7952fd9e-d112-4bba-92fc-24481c11f6a2: !Template answer_choices: A ||| B - id: d6f53c48-9d17-440f-bc3d-6d01005d2ab4 + id: 7952fd9e-d112-4bba-92fc-24481c11f6a2 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} @@ -263,29 +207,15 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B - reference: '' - d81c7f05-20d8-459d-8357-4206356e92b9: !Template - answer_choices: Yes ||| No - id: d81c7f05-20d8-459d-8357-4206356e92b9 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes + name: grammatical_between_A_B reference: '' - febf1477-3dc8-4b8a-a024-8f32fd63d084: !Template + a745579b-fb13-456b-a14d-6a23e3d875f9: !Template answer_choices: A ||| B - id: febf1477-3dc8-4b8a-a024-8f32fd63d084 + id: a745579b-fb13-456b-a14d-6a23e3d875f9 jinja: '{% set shuffled_order = [0, 1] | random %} + {% set shuffled_order_options = [0, 1] | random %} + Read the following two sentences. {% if shuffled_order == 0 %} @@ -302,8 +232,8 @@ templates: {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -317,5 +247,16 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: better_A_B + reference: '' + b36f421a-33d6-4e50-927e-01d5f75d8461: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: b36f421a-33d6-4e50-927e-01d5f75d8461 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt reference: '' diff --git a/promptsource/templates/blimp/animate_subject_trans/templates.yaml b/promptsource/templates/blimp/animate_subject_trans/templates.yaml index 3e2cf7cdb..477cb7b90 100644 --- a/promptsource/templates/blimp/animate_subject_trans/templates.yaml +++ b/promptsource/templates/blimp/animate_subject_trans/templates.yaml @@ -1,109 +1,33 @@ dataset: blimp subset: animate_subject_trans templates: - 064c8d03-21f3-4c53-8b38-13068809707a: !Template + 0e668f2a-2d52-4619-9225-8a934defb775: !Template answer_choices: Yes ||| No - id: 064c8d03-21f3-4c53-8b38-13068809707a - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes - reference: '' - 1c3fba07-ae44-4318-a279-79b08df8e017: !Template - answer_choices: Yes ||| No - id: 1c3fba07-ae44-4318-a279-79b08df8e017 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no - reference: '' - 1ddd4097-a296-4d71-aaab-906b57edafe9: !Template - answer_choices: Yes ||| No - id: 1ddd4097-a296-4d71-aaab-906b57edafe9 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes - reference: '' - 2adf5f8f-8bd0-4ca1-b916-decedf6edc46: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 2adf5f8f-8bd0-4ca1-b916-decedf6edc46 - jinja: '" ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: null_prompt - reference: '' - 2b6b48d3-cb8e-4ea4-ad31-3285e40c7a03: !Template - answer_choices: 1 ||| 2 - id: 2b6b48d3-cb8e-4ea4-ad31-3285e40c7a03 + id: 0e668f2a-2d52-4619-9225-8a934defb775 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. - - {% if shuffled_order == 0 %} - - 1: {{ sentence_good }} - - 2: {{ sentence_bad }} - - {% else %} - - 1: {{ sentence_bad }} - - 2: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - {{ answer_choices[0] }} + {% set shuffled_order_options = [0, 1] | random %} - {% else %} + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" - {{ answer_choices[1] }} - - {% endif %}' + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: grammatical_between_1_2 + original_task: false + name: single_sentence_good_yes_no reference: '' - 2e8da105-49ba-4730-b0fa-1736d634d8d8: !Template + 20018611-aea5-4866-b3b7-74ec0b68920e: !Template answer_choices: A ||| B - id: 2e8da105-49ba-4730-b0fa-1736d634d8d8 + id: 20018611-aea5-4866-b3b7-74ec0b68920e jinja: '{% set shuffled_order = [0, 1] | random %} + {% set shuffled_order_options = [0, 1] | random %} + Read the following two sentences. {% if shuffled_order == 0 %} @@ -120,8 +44,8 @@ templates: {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -137,29 +61,33 @@ templates: original_task: true name: better_A_B reference: '' - 5c1a565e-a0d4-408f-b215-8544a754894d: !Template + 3dda3756-4efa-4f0a-a772-1a41262e5476: !Template answer_choices: A ||| B - id: 5c1a565e-a0d4-408f-b215-8544a754894d + id: 3dda3756-4efa-4f0a-a772-1a41262e5476 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -173,14 +101,17 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: grammatical_which_one_A_B reference: '' - 73297481-c7b1-4998-8496-7fcdcc45b98c: !Template + 9e0965c1-c2bd-4dc1-bf8c-f310d73f5758: !Template answer_choices: 1 ||| 2 - id: 73297481-c7b1-4998-8496-7fcdcc45b98c + id: 9e0965c1-c2bd-4dc1-bf8c-f310d73f5758 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} @@ -214,39 +145,26 @@ templates: original_task: true name: grammatical_which_one_1_2 reference: '' - 7903aba0-972f-4568-bed5-9318708b860f: !Template + dc62e655-3192-49f7-ace1-7b14732fab8c: !Template answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 7903aba0-972f-4568-bed5-9318708b860f + id: dc62e655-3192-49f7-ace1-7b14732fab8c jinja: ' ||| {{ sentence_good }}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - Accuracy original_task: true - name: true_null_prompt - reference: '' - 9171e40d-c652-43c7-9048-b62f2f64fe35: !Template - answer_choices: Yes ||| No - id: 9171e40d-c652-43c7-9048-b62f2f64fe35 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no + name: null_prompt reference: '' - 9b16b800-abea-4b12-91a9-361c6ab05ce6: !Template + e91b1182-854e-4473-b9f2-8f68c4d72c8c: !Template answer_choices: A ||| B - id: 9b16b800-abea-4b12-91a9-361c6ab05ce6 + id: e91b1182-854e-4473-b9f2-8f68c4d72c8c jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} @@ -280,24 +198,27 @@ templates: original_task: true name: grammatical_between_A_B reference: '' - f874c53d-e05d-4ee2-81cf-b01968e4033f: !Template - answer_choices: A ||| B - id: f874c53d-e05d-4ee2-81cf-b01968e4033f + ebe63f74-9c6f-4a29-9fb1-2567c5354609: !Template + answer_choices: 1 ||| 2 + id: ebe63f74-9c6f-4a29-9fb1-2567c5354609 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -317,5 +238,25 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: grammatical_between_1_2 + reference: '' + f7936b93-8a2e-4ae6-b85f-a494ed49a1b3: !Template + answer_choices: Yes ||| No + id: f7936b93-8a2e-4ae6-b85f-a494ed49a1b3 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no reference: '' diff --git a/promptsource/templates/blimp/causative/templates.yaml b/promptsource/templates/blimp/causative/templates.yaml index 3107eb646..9994ae8db 100644 --- a/promptsource/templates/blimp/causative/templates.yaml +++ b/promptsource/templates/blimp/causative/templates.yaml @@ -1,28 +1,46 @@ dataset: blimp subset: causative templates: - 168f4a04-eabd-44f5-80b6-9599b90f0911: !Template + 0107b558-5748-4807-9238-9dc921b36220: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 0107b558-5748-4807-9238-9dc921b36220 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + 0a3230e8-2cce-4c41-aaef-eee10cb200e3: !Template answer_choices: Yes ||| No - id: 168f4a04-eabd-44f5-80b6-9599b90f0911 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + id: 0a3230e8-2cce-4c41-aaef-eee10cb200e3 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_bad_no_yes + name: single_sentence_good_yes_no reference: '' - 17de8f10-c62f-42d7-a800-fb34e70a9c3a: !Template + 2a1d7c07-be9e-4980-81e3-f36c4453b834: !Template answer_choices: 1 ||| 2 - id: 17de8f10-c62f-42d7-a800-fb34e70a9c3a + id: 2a1d7c07-be9e-4980-81e3-f36c4453b834 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} @@ -54,26 +72,29 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: grammatical_which_one_1_2 reference: '' - 291a8522-60ac-4634-8fb9-5fbbf25154f2: !Template - answer_choices: A ||| B - id: 291a8522-60ac-4634-8fb9-5fbbf25154f2 + 596e688d-2267-47e1-a427-18e256798411: !Template + answer_choices: 1 ||| 2 + id: 596e688d-2267-47e1-a427-18e256798411 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -93,42 +114,35 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B - reference: '' - 29d6a88c-b38d-4763-b83a-cb5e668ada75: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 29d6a88c-b38d-4763-b83a-cb5e668ada75 - jinja: ' ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: true_null_prompt + name: grammatical_between_1_2 reference: '' - 749bf8fc-7aba-4a96-a080-3bc2498c3589: !Template + 6b8a30a4-d070-4029-b93d-8f18c9247140: !Template answer_choices: A ||| B - id: 749bf8fc-7aba-4a96-a080-3bc2498c3589 + id: 6b8a30a4-d070-4029-b93d-8f18c9247140 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -142,57 +156,37 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B - reference: '' - 8249180d-bd45-4e40-8fe9-af9c44faeaa9: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 8249180d-bd45-4e40-8fe9-af9c44faeaa9 - jinja: '" ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: null_prompt + name: grammatical_which_one_A_B reference: '' - 9fad506f-59eb-41b9-823e-8840b1051d34: !Template + b3bd8baf-0fcb-4d47-91fd-72bdbce631de: !Template answer_choices: Yes ||| No - id: 9fad506f-59eb-41b9-823e-8840b1051d34 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + id: b3bd8baf-0fcb-4d47-91fd-72bdbce631de + jinja: '{% set shuffled_order = [0, 1] | random %} - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no - reference: '' - c0e592e1-f655-4246-87b0-a63a66d90609: !Template - answer_choices: Yes ||| No - id: c0e592e1-f655-4246-87b0-a63a66d90609 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_good_no_yes + name: single_sentence_bad_yes_no reference: '' - e4d278e5-1aa8-44e4-a56a-92261dedd5e9: !Template + c7852c5e-08be-4672-b534-b560099db2af: !Template answer_choices: A ||| B - id: e4d278e5-1aa8-44e4-a56a-92261dedd5e9 + id: c7852c5e-08be-4672-b534-b560099db2af jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} @@ -226,11 +220,13 @@ templates: original_task: true name: grammatical_between_A_B reference: '' - e8f0e6c3-43e4-4351-b590-482b6651173e: !Template + f53e9bec-4fca-44cb-aa76-bf245d3f9fb7: !Template answer_choices: A ||| B - id: e8f0e6c3-43e4-4351-b590-482b6651173e + id: f53e9bec-4fca-44cb-aa76-bf245d3f9fb7 jinja: '{% set shuffled_order = [0, 1] | random %} + {% set shuffled_order_options = [0, 1] | random %} + Read the following two sentences. {% if shuffled_order == 0 %} @@ -247,8 +243,8 @@ templates: {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -262,60 +258,5 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A - reference: '' - ef9fd9f5-1656-4251-80d2-5f79e91211fc: !Template - answer_choices: 1 ||| 2 - id: ef9fd9f5-1656-4251-80d2-5f79e91211fc - jinja: '{% set shuffled_order = [0, 1] | random %} - - Which one of the following sentences is grammatical? Please answer 1 or 2. - - {% if shuffled_order == 0 %} - - 1: {{ sentence_good }} - - 2: {{ sentence_bad }} - - {% else %} - - 1: {{ sentence_bad }} - - 2: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} - - {% endif %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: grammatical_which_one_1_2 - reference: '' - f14fbe66-2bfb-4863-bb97-6c8aba7c290a: !Template - answer_choices: Yes ||| No - id: f14fbe66-2bfb-4863-bb97-6c8aba7c290a - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no + name: better_A_B reference: '' diff --git a/promptsource/templates/blimp/complex_NP_island/templates.yaml b/promptsource/templates/blimp/complex_NP_island/templates.yaml index 40b4ea0ae..7fdc8291a 100644 --- a/promptsource/templates/blimp/complex_NP_island/templates.yaml +++ b/promptsource/templates/blimp/complex_NP_island/templates.yaml @@ -1,24 +1,27 @@ dataset: blimp subset: complex_NP_island templates: - 10dfe420-443f-49aa-9022-af2dd3d270bb: !Template - answer_choices: 1 ||| 2 - id: 10dfe420-443f-49aa-9022-af2dd3d270bb + 37f9405d-9482-4691-8853-945a22210b9e: !Template + answer_choices: A ||| B + id: 37f9405d-9482-4691-8853-945a22210b9e jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -38,42 +41,55 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: grammatical_between_A_B reference: '' - 2699352c-8851-44a5-9e53-2e1f7d047fdd: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 2699352c-8851-44a5-9e53-2e1f7d047fdd - jinja: ' ||| {{ sentence_good }}' + 3cd258c4-b303-4884-9db4-4ef3d89a1ef2: !Template + answer_choices: Yes ||| No + id: 3cd258c4-b303-4884-9db4-4ef3d89a1ef2 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata - choices_in_prompt: false + choices_in_prompt: true metrics: - Accuracy - original_task: true - name: true_null_prompt + original_task: false + name: single_sentence_bad_yes_no reference: '' - 44ec48a9-bf3a-4a40-9482-e22342b17ef5: !Template + 3e4570a1-bf6c-43e1-9566-6054587f9203: !Template answer_choices: A ||| B - id: 44ec48a9-bf3a-4a40-9482-e22342b17ef5 + id: 3e4570a1-bf6c-43e1-9566-6054587f9203 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -87,26 +103,29 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: grammatical_which_one_A_B reference: '' - 5d8020db-d93c-4d65-806e-1f6974e23393: !Template - answer_choices: A ||| B - id: 5d8020db-d93c-4d65-806e-1f6974e23393 + 6d115be5-5911-48ef-ae43-88b92defca66: !Template + answer_choices: 1 ||| 2 + id: 6d115be5-5911-48ef-ae43-88b92defca66 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -126,29 +145,46 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: grammatical_which_one_1_2 reference: '' - 642a5a47-bd13-4c2a-8948-a371520549fb: !Template + 6ed7e579-3927-4646-890d-1c0ebc2ffc60: !Template answer_choices: Yes ||| No - id: 642a5a47-bd13-4c2a-8948-a371520549fb - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + id: 6ed7e579-3927-4646-890d-1c0ebc2ffc60 + jinja: '{% set shuffled_order = [0, 1] | random %} - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_bad_yes_no + name: single_sentence_good_yes_no reference: '' - 75916a91-d763-43dc-8e0f-b43efd72e2e4: !Template + 9c282484-a3da-47c4-afcd-6363d744f558: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 9c282484-a3da-47c4-afcd-6363d744f558 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + b3f7db9b-5363-4076-a966-ef6b17d90991: !Template answer_choices: A ||| B - id: 75916a91-d763-43dc-8e0f-b43efd72e2e4 + id: b3f7db9b-5363-4076-a966-ef6b17d90991 jinja: '{% set shuffled_order = [0, 1] | random %} + {% set shuffled_order_options = [0, 1] | random %} + Read the following two sentences. {% if shuffled_order == 0 %} @@ -165,8 +201,8 @@ templates: {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -182,78 +218,15 @@ templates: original_task: true name: better_A_B reference: '' - 838919e3-8e61-4843-8469-dfe7ddab9027: !Template - answer_choices: A ||| B - id: 838919e3-8e61-4843-8469-dfe7ddab9027 - jinja: '{% set shuffled_order = [0, 1] | random %} - - Which one of the following sentences is grammatical? Please answer A or B. - - {% if shuffled_order == 0 %} - - A: {{ sentence_good }} - - B: {{ sentence_bad }} - - {% else %} - - A: {{ sentence_bad }} - - B: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} - - {% endif %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: grammatical_which_one_A_B - reference: '' - 986f3bef-10b5-47b2-873c-950f3e5f1d8e: !Template - answer_choices: Yes ||| No - id: 986f3bef-10b5-47b2-873c-950f3e5f1d8e - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes - reference: '' - 9d32b8a0-58eb-4f5d-97d8-32bc5e69484f: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 9d32b8a0-58eb-4f5d-97d8-32bc5e69484f - jinja: '" ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: null_prompt - reference: '' - 9f96111f-2282-4c40-9110-b0dadc97297b: !Template + e12a2257-9641-47a4-bcaf-d674e35b77e8: !Template answer_choices: 1 ||| 2 - id: 9f96111f-2282-4c40-9110-b0dadc97297b + id: e12a2257-9641-47a4-bcaf-d674e35b77e8 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} @@ -285,37 +258,5 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 - reference: '' - ca8ab9ef-b864-41a5-acc7-4ae8064c6583: !Template - answer_choices: Yes ||| No - id: ca8ab9ef-b864-41a5-acc7-4ae8064c6583 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no - reference: '' - e7f0c3da-177c-492c-aa3f-729e7059ef21: !Template - answer_choices: Yes ||| No - id: e7f0c3da-177c-492c-aa3f-729e7059ef21 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes + name: grammatical_between_1_2 reference: '' diff --git a/promptsource/templates/blimp/coordinate_structure_constraint_complex_left_branch/templates.yaml b/promptsource/templates/blimp/coordinate_structure_constraint_complex_left_branch/templates.yaml index 2bc5b7cd4..c44a45b3d 100644 --- a/promptsource/templates/blimp/coordinate_structure_constraint_complex_left_branch/templates.yaml +++ b/promptsource/templates/blimp/coordinate_structure_constraint_complex_left_branch/templates.yaml @@ -1,63 +1,47 @@ dataset: blimp subset: coordinate_structure_constraint_complex_left_branch templates: - 1dc0a548-237a-4bfd-9c72-dbfab9344488: !Template - answer_choices: 1 ||| 2 - id: 1dc0a548-237a-4bfd-9c72-dbfab9344488 + 0e93a8db-4f93-4ac3-9502-6a8132c2a647: !Template + answer_choices: Yes ||| No + id: 0e93a8db-4f93-4ac3-9502-6a8132c2a647 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. - - {% if shuffled_order == 0 %} - - 1: {{ sentence_good }} - - 2: {{ sentence_bad }} - - {% else %} - - 1: {{ sentence_bad }} - - 2: {{ sentence_good }} - - {% endif %} - - ||| + {% set shuffled_order_options = [0, 1] | random %} - {% if shuffled_order == 0 %} + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} - - {% endif %}' + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: grammatical_between_1_2 + original_task: false + name: single_sentence_good_yes_no reference: '' - 266fdc02-5341-4e31-b5ae-879e7954e6ae: !Template - answer_choices: A ||| B - id: 266fdc02-5341-4e31-b5ae-879e7954e6ae + 1630a692-c39a-447f-a672-9c441fffd3e3: !Template + answer_choices: 1 ||| 2 + id: 1630a692-c39a-447f-a672-9c441fffd3e3 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -77,77 +61,15 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B - reference: '' - 2be386e3-f37a-42fa-8d87-322ef523dfb8: !Template - answer_choices: Yes ||| No - id: 2be386e3-f37a-42fa-8d87-322ef523dfb8 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no - reference: '' - 5bd12502-5a68-47d9-871c-53c35d202dbb: !Template - answer_choices: Yes ||| No - id: 5bd12502-5a68-47d9-871c-53c35d202dbb - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes - reference: '' - 5df8f7b5-014d-4478-b300-355f71a1b029: !Template - answer_choices: Yes ||| No - id: 5df8f7b5-014d-4478-b300-355f71a1b029 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no - reference: '' - 7baef85f-4114-4d4d-9f2a-90d38675d5aa: !Template - answer_choices: Yes ||| No - id: 7baef85f-4114-4d4d-9f2a-90d38675d5aa - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes + name: grammatical_which_one_1_2 reference: '' - 89c9a342-28a5-4d6e-935a-03c08c02cfbf: !Template + 29b55f60-5b70-4dc2-93fd-e03ce2e0b8cc: !Template answer_choices: A ||| B - id: 89c9a342-28a5-4d6e-935a-03c08c02cfbf + id: 29b55f60-5b70-4dc2-93fd-e03ce2e0b8cc jinja: '{% set shuffled_order = [0, 1] | random %} + {% set shuffled_order_options = [0, 1] | random %} + Read the following two sentences. {% if shuffled_order == 0 %} @@ -164,8 +86,8 @@ templates: {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -181,12 +103,15 @@ templates: original_task: true name: better_A_B reference: '' - 8efe286f-814b-42c0-925d-6ddc7c7cb607: !Template + 3e2be3c5-e21b-4cbd-9bdb-a951f698de92: !Template answer_choices: 1 ||| 2 - id: 8efe286f-814b-42c0-925d-6ddc7c7cb607 + id: 3e2be3c5-e21b-4cbd-9bdb-a951f698de92 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} @@ -218,36 +143,17 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 - reference: '' - b8c2ed75-8b30-4962-b02d-1638413c8d14: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: b8c2ed75-8b30-4962-b02d-1638413c8d14 - jinja: '" ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: null_prompt - reference: '' - bd7b9332-c8ac-4626-9484-b94644bd4569: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: bd7b9332-c8ac-4626-9484-b94644bd4569 - jinja: ' ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: true_null_prompt + name: grammatical_between_1_2 reference: '' - c4045c0d-0dd3-4370-96a9-0d5da4b92ebb: !Template + 4006ea8e-e9ec-4605-b230-0e9a2415eb66: !Template answer_choices: A ||| B - id: c4045c0d-0dd3-4370-96a9-0d5da4b92ebb + id: 4006ea8e-e9ec-4605-b230-0e9a2415eb66 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} @@ -281,29 +187,64 @@ templates: original_task: true name: grammatical_between_A_B reference: '' - e21c0a63-bfaa-4b6c-8be6-7cd2ca9e7a7f: !Template + 89ca822d-5cf7-4362-aadc-df69544ef4e3: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 89ca822d-5cf7-4362-aadc-df69544ef4e3 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + 90da64af-ad98-4229-a2d7-b04191f55b1c: !Template + answer_choices: Yes ||| No + id: 90da64af-ad98-4229-a2d7-b04191f55b1c + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + fe0a2557-7de0-4b98-b2cb-28aa03d33e44: !Template answer_choices: A ||| B - id: e21c0a63-bfaa-4b6c-8be6-7cd2ca9e7a7f + id: fe0a2557-7de0-4b98-b2cb-28aa03d33e44 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -317,5 +258,5 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: grammatical_which_one_A_B reference: '' diff --git a/promptsource/templates/blimp/coordinate_structure_constraint_object_extraction/templates.yaml b/promptsource/templates/blimp/coordinate_structure_constraint_object_extraction/templates.yaml index b68636d7e..c5917839c 100644 --- a/promptsource/templates/blimp/coordinate_structure_constraint_object_extraction/templates.yaml +++ b/promptsource/templates/blimp/coordinate_structure_constraint_object_extraction/templates.yaml @@ -1,68 +1,31 @@ dataset: blimp subset: coordinate_structure_constraint_object_extraction templates: - 1c920ce5-a4e1-4429-ad69-cb75b85c2ee5: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 1c920ce5-a4e1-4429-ad69-cb75b85c2ee5 - jinja: '" ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: null_prompt - reference: '' - 1d6280db-3f10-476d-b64d-a6975f00717c: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 1d6280db-3f10-476d-b64d-a6975f00717c - jinja: ' ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: true_null_prompt - reference: '' - 26d3df71-b001-41f1-b3c4-a8f94ecc8bc8: !Template - answer_choices: Yes ||| No - id: 26d3df71-b001-41f1-b3c4-a8f94ecc8bc8 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no - reference: '' - 288a3c31-0441-41f5-bead-d1e5ebb8a931: !Template - answer_choices: 1 ||| 2 - id: 288a3c31-0441-41f5-bead-d1e5ebb8a931 + 048bbd16-16bd-4aa3-a0cb-f65d38eb5193: !Template + answer_choices: A ||| B + id: 048bbd16-16bd-4aa3-a0cb-f65d38eb5193 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -76,26 +39,49 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: better_A_B reference: '' - 4178601f-7e50-4fd1-9c2f-c9eebe0ce75c: !Template - answer_choices: 1 ||| 2 - id: 4178601f-7e50-4fd1-9c2f-c9eebe0ce75c + 351916b8-be3d-45d0-b7ff-1e2c8f317449: !Template + answer_choices: Yes ||| No + id: 351916b8-be3d-45d0-b7ff-1e2c8f317449 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 47d445a6-5a8e-428e-8520-b0084d46e2ca: !Template + answer_choices: A ||| B + id: 47d445a6-5a8e-428e-8520-b0084d46e2ca jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -115,42 +101,29 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 - reference: '' - 5b98d834-0256-41ec-9aae-e761013fa8fd: !Template - answer_choices: Yes ||| No - id: 5b98d834-0256-41ec-9aae-e761013fa8fd - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes + name: grammatical_which_one_A_B reference: '' - 84105f1d-256a-4c3f-8cd8-eb26e8393283: !Template - answer_choices: A ||| B - id: 84105f1d-256a-4c3f-8cd8-eb26e8393283 + 73858b20-2880-4fd6-9c44-abd3c0af91cd: !Template + answer_choices: 1 ||| 2 + id: 73858b20-2880-4fd6-9c44-abd3c0af91cd jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -170,31 +143,35 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: grammatical_between_1_2 reference: '' - 8c227511-2cbb-4b9d-83c5-cd5f7143f123: !Template - answer_choices: A ||| B - id: 8c227511-2cbb-4b9d-83c5-cd5f7143f123 + a1e4dafa-66b6-49b1-806e-a53ab60aeb73: !Template + answer_choices: 1 ||| 2 + id: a1e4dafa-66b6-49b1-806e-a53ab60aeb73 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -208,46 +185,17 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A - reference: '' - 8f521740-cd3b-414e-9a14-d9b832ecd09d: !Template - answer_choices: Yes ||| No - id: 8f521740-cd3b-414e-9a14-d9b832ecd09d - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no - reference: '' - 97ca2291-9977-4526-abdf-a4baafea371e: !Template - answer_choices: Yes ||| No - id: 97ca2291-9977-4526-abdf-a4baafea371e - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes + name: grammatical_which_one_1_2 reference: '' - bfb80d33-9748-46e4-bc20-f02ca0ed5681: !Template + ae2b3124-dd9d-4b78-a2da-f9168345fd99: !Template answer_choices: A ||| B - id: bfb80d33-9748-46e4-bc20-f02ca0ed5681 + id: ae2b3124-dd9d-4b78-a2da-f9168345fd99 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} @@ -279,43 +227,36 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: grammatical_between_A_B reference: '' - cbb7b2e0-9a13-449d-ad23-2fb3247dc78d: !Template - answer_choices: A ||| B - id: cbb7b2e0-9a13-449d-ad23-2fb3247dc78d + b4a33a5f-cc31-4366-a84d-e97b8129a144: !Template + answer_choices: Yes ||| No + id: b4a33a5f-cc31-4366-a84d-e97b8129a144 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. - - {% if shuffled_order == 0 %} - - Sentence A: {{ sentence_good }} - - Sentence B: {{ sentence_bad }} - - {% else %} + {% set shuffled_order_options = [0, 1] | random %} - Sentence A: {{ sentence_bad }} - - Sentence B: {{ sentence_good }} - - {% endif %} + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} - - {% endif %}' + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + c2c9b04a-5158-4d0a-b170-17c0cf8d52de: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: c2c9b04a-5158-4d0a-b170-17c0cf8d52de + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy original_task: true - name: better_A_B + name: null_prompt reference: '' diff --git a/promptsource/templates/blimp/determiner_noun_agreement_1/templates.yaml b/promptsource/templates/blimp/determiner_noun_agreement_1/templates.yaml index 66a848ff2..47020325b 100644 --- a/promptsource/templates/blimp/determiner_noun_agreement_1/templates.yaml +++ b/promptsource/templates/blimp/determiner_noun_agreement_1/templates.yaml @@ -1,93 +1,15 @@ dataset: blimp subset: determiner_noun_agreement_1 templates: - 1db52d9a-70f0-44c8-a6e6-77fbfa6a9a18: !Template - answer_choices: Yes ||| No - id: 1db52d9a-70f0-44c8-a6e6-77fbfa6a9a18 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes - reference: '' - 29da8721-7fb9-44e5-8b47-5aaba62bfc4b: !Template - answer_choices: Yes ||| No - id: 29da8721-7fb9-44e5-8b47-5aaba62bfc4b - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no - reference: '' - 45aa0402-d965-48e9-8cd5-a44f3176172f: !Template - answer_choices: A ||| B - id: 45aa0402-d965-48e9-8cd5-a44f3176172f - jinja: '{% set shuffled_order = [0, 1] | random %} - - Read the following two sentences. - - {% if shuffled_order == 0 %} - - Sentence A: {{ sentence_good }} - - Sentence B: {{ sentence_bad }} - - {% else %} - - Sentence A: {{ sentence_bad }} - - Sentence B: {{ sentence_good }} - - {% endif %} - - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} - - {% endif %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: better_B_A - reference: '' - 4a2b1c8f-a224-49b8-9ed7-af5b74ef6a7f: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 4a2b1c8f-a224-49b8-9ed7-af5b74ef6a7f - jinja: '" ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: null_prompt - reference: '' - 55ad36f6-af14-4cdc-80fa-661aad9b38b3: !Template + 03d16d6f-bfdd-4b84-919f-e6c6b81f3c88: !Template answer_choices: 1 ||| 2 - id: 55ad36f6-af14-4cdc-80fa-661aad9b38b3 + id: 03d16d6f-bfdd-4b84-919f-e6c6b81f3c88 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} @@ -119,14 +41,17 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: grammatical_between_1_2 reference: '' - 5c4eb1e0-370b-49cd-8399-fb6296582eee: !Template + 23ebe991-1aeb-4873-b9d5-c5235cf886c6: !Template answer_choices: A ||| B - id: 5c4eb1e0-370b-49cd-8399-fb6296582eee + id: 23ebe991-1aeb-4873-b9d5-c5235cf886c6 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} @@ -160,12 +85,66 @@ templates: original_task: true name: grammatical_between_A_B reference: '' - 64321690-5afb-428d-8339-67415356f5f9: !Template + 24f8db17-2c78-44a2-b8b9-db2237818623: !Template + answer_choices: Yes ||| No + id: 24f8db17-2c78-44a2-b8b9-db2237818623 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 341495b5-cf05-4c77-9678-56c5039ca446: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 341495b5-cf05-4c77-9678-56c5039ca446 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + 4d771189-3657-43f0-bf82-984ad14ebfa8: !Template + answer_choices: Yes ||| No + id: 4d771189-3657-43f0-bf82-984ad14ebfa8 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 89eed9d7-5e70-4948-b39b-4bd03b2ffc5f: !Template answer_choices: 1 ||| 2 - id: 64321690-5afb-428d-8339-67415356f5f9 + id: 89eed9d7-5e70-4948-b39b-4bd03b2ffc5f jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} @@ -197,29 +176,15 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 - reference: '' - a069a35b-c357-469d-b307-d92a57e3fde7: !Template - answer_choices: Yes ||| No - id: a069a35b-c357-469d-b307-d92a57e3fde7 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no + name: grammatical_which_one_1_2 reference: '' - b241bf1a-e1ce-4f65-8962-7b176760dce1: !Template + 8a9e9fae-a83d-456f-8b5b-df688ee1bad2: !Template answer_choices: A ||| B - id: b241bf1a-e1ce-4f65-8962-7b176760dce1 + id: 8a9e9fae-a83d-456f-8b5b-df688ee1bad2 jinja: '{% set shuffled_order = [0, 1] | random %} + {% set shuffled_order_options = [0, 1] | random %} + Read the following two sentences. {% if shuffled_order == 0 %} @@ -236,8 +201,8 @@ templates: {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -253,12 +218,15 @@ templates: original_task: true name: better_A_B reference: '' - b71c1107-873d-4f7b-a55d-2d421504b849: !Template + f8850ec9-845d-4271-b691-71068a0f45f9: !Template answer_choices: A ||| B - id: b71c1107-873d-4f7b-a55d-2d421504b849 + id: f8850ec9-845d-4271-b691-71068a0f45f9 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} @@ -292,30 +260,3 @@ templates: original_task: true name: grammatical_which_one_A_B reference: '' - f6fefaec-b1f8-4543-97c4-282a541219f4: !Template - answer_choices: Yes ||| No - id: f6fefaec-b1f8-4543-97c4-282a541219f4 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes - reference: '' - f80aed19-f670-4722-9513-efb0d7c654c7: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: f80aed19-f670-4722-9513-efb0d7c654c7 - jinja: ' ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: true_null_prompt - reference: '' diff --git a/promptsource/templates/blimp/determiner_noun_agreement_2/templates.yaml b/promptsource/templates/blimp/determiner_noun_agreement_2/templates.yaml index cc603166c..5b3a03415 100644 --- a/promptsource/templates/blimp/determiner_noun_agreement_2/templates.yaml +++ b/promptsource/templates/blimp/determiner_noun_agreement_2/templates.yaml @@ -1,79 +1,58 @@ dataset: blimp subset: determiner_noun_agreement_2 templates: - 124c66e5-f4fe-44a6-9942-0fb38a0b3f5e: !Template - answer_choices: A ||| B - id: 124c66e5-f4fe-44a6-9942-0fb38a0b3f5e - jinja: '{% set shuffled_order = [0, 1] | random %} - - Between the two sentences, which one is grammatical? Please answer A or B. - - {% if shuffled_order == 0 %} - - A: {{ sentence_good }} - - B: {{ sentence_bad }} - - {% else %} - - A: {{ sentence_bad }} - - B: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} - - {% endif %}' + 1ff49155-9e74-463f-9bea-5ea0f1c4c5cc: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 1ff49155-9e74-463f-9bea-5ea0f1c4c5cc + jinja: ' ||| {{ sentence_good }}' metadata: !TemplateMetadata - choices_in_prompt: true + choices_in_prompt: false metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: null_prompt reference: '' - 15d41b02-c038-4b99-9ea3-1b9696dc30e5: !Template + 49f63f3b-5f6a-4663-a4f3-6e419e2c8e46: !Template answer_choices: Yes ||| No - id: 15d41b02-c038-4b99-9ea3-1b9696dc30e5 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + id: 49f63f3b-5f6a-4663-a4f3-6e419e2c8e46 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_good_yes_no + name: single_sentence_bad_yes_no reference: '' - 22129942-86ae-4290-93b4-2ce9cceb2503: !Template - answer_choices: 1 ||| 2 - id: 22129942-86ae-4290-93b4-2ce9cceb2503 + 4b2714f1-b970-4f26-9893-89a4b887260f: !Template + answer_choices: A ||| B + id: 4b2714f1-b970-4f26-9893-89a4b887260f jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -93,13 +72,15 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: grammatical_between_A_B reference: '' - 31377a83-cca7-433f-a727-ab3e3b1378cb: !Template + 6a002670-0148-485a-9278-e6951bd60409: !Template answer_choices: A ||| B - id: 31377a83-cca7-433f-a727-ab3e3b1378cb + id: 6a002670-0148-485a-9278-e6951bd60409 jinja: '{% set shuffled_order = [0, 1] | random %} + {% set shuffled_order_options = [0, 1] | random %} + Read the following two sentences. {% if shuffled_order == 0 %} @@ -116,8 +97,8 @@ templates: {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -133,12 +114,35 @@ templates: original_task: true name: better_A_B reference: '' - 3d2bfdcc-50a4-4d36-8bce-3dd109386147: !Template + b2e18f6d-69c2-414c-b36f-01eadcd23f39: !Template + answer_choices: Yes ||| No + id: b2e18f6d-69c2-414c-b36f-01eadcd23f39 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + ca5ce2c1-4690-49d4-a18d-53431b9b2a1d: !Template answer_choices: 1 ||| 2 - id: 3d2bfdcc-50a4-4d36-8bce-3dd109386147 + id: ca5ce2c1-4690-49d4-a18d-53431b9b2a1d jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} @@ -170,69 +174,29 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 - reference: '' - 4f70018c-1108-4801-b6a1-161c71bce8b3: !Template - answer_choices: Yes ||| No - id: 4f70018c-1108-4801-b6a1-161c71bce8b3 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no - reference: '' - 66f08583-4be1-4d57-9c01-6e8077610323: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 66f08583-4be1-4d57-9c01-6e8077610323 - jinja: ' ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: true_null_prompt - reference: '' - 8b6c24e4-5a25-4fbf-b396-d988e4eee38f: !Template - answer_choices: Yes ||| No - id: 8b6c24e4-5a25-4fbf-b396-d988e4eee38f - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes + name: grammatical_between_1_2 reference: '' - 9a6ec137-1fb3-4f6d-96e0-18bc5aaa2e34: !Template - answer_choices: A ||| B - id: 9a6ec137-1fb3-4f6d-96e0-18bc5aaa2e34 + e8c2e7c6-9197-44c8-a14c-4bac6fa7df44: !Template + answer_choices: 1 ||| 2 + id: e8c2e7c6-9197-44c8-a14c-4bac6fa7df44 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -252,47 +216,35 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B - reference: '' - a870aa59-a74c-4522-81e3-48f5d35b07a7: !Template - answer_choices: Yes ||| No - id: a870aa59-a74c-4522-81e3-48f5d35b07a7 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes + name: grammatical_which_one_1_2 reference: '' - ba8b3eda-b858-4d24-b1a2-7e8f6cdac276: !Template + ee66157f-f99e-4431-bfdb-cf041ac21c52: !Template answer_choices: A ||| B - id: ba8b3eda-b858-4d24-b1a2-7e8f6cdac276 + id: ee66157f-f99e-4431-bfdb-cf041ac21c52 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -306,16 +258,5 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A - reference: '' - dd61ea3c-4770-41ba-9be7-e6803a91433d: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: dd61ea3c-4770-41ba-9be7-e6803a91433d - jinja: '" ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: null_prompt + name: grammatical_which_one_A_B reference: '' diff --git a/promptsource/templates/blimp/determiner_noun_agreement_irregular_1/templates.yaml b/promptsource/templates/blimp/determiner_noun_agreement_irregular_1/templates.yaml index 0ac826893..67072388b 100644 --- a/promptsource/templates/blimp/determiner_noun_agreement_irregular_1/templates.yaml +++ b/promptsource/templates/blimp/determiner_noun_agreement_irregular_1/templates.yaml @@ -1,12 +1,26 @@ dataset: blimp subset: determiner_noun_agreement_irregular_1 templates: - 2cc9d0e0-59c4-4140-8120-ec667040b627: !Template + 049b2393-dd62-4ce4-bba4-ee0afc7c3ec4: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 049b2393-dd62-4ce4-bba4-ee0afc7c3ec4 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + 3774724e-fa89-427f-acfe-e176cf2459a4: !Template answer_choices: 1 ||| 2 - id: 2cc9d0e0-59c4-4140-8120-ec667040b627 + id: 3774724e-fa89-427f-acfe-e176cf2459a4 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} @@ -40,29 +54,33 @@ templates: original_task: true name: grammatical_between_1_2 reference: '' - 2e41c457-200f-47a1-97f6-fd09a0ab83ed: !Template - answer_choices: A ||| B - id: 2e41c457-200f-47a1-97f6-fd09a0ab83ed + 41f2912c-1126-4940-91f1-3b2b837d8460: !Template + answer_choices: 1 ||| 2 + id: 41f2912c-1126-4940-91f1-3b2b837d8460 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -76,57 +94,17 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A - reference: '' - 9307601d-e073-4bed-be6d-67a9d39f45f3: !Template - answer_choices: Yes ||| No - id: 9307601d-e073-4bed-be6d-67a9d39f45f3 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no - reference: '' - 9c2a4478-363d-4caf-9a6a-713dd0f3c86e: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 9c2a4478-363d-4caf-9a6a-713dd0f3c86e - jinja: '" ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: null_prompt - reference: '' - c8609ae6-0f99-4c4a-8f48-aaebbbc9c362: !Template - answer_choices: Yes ||| No - id: c8609ae6-0f99-4c4a-8f48-aaebbbc9c362 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes + name: grammatical_which_one_1_2 reference: '' - c8cedf93-50df-490d-9f37-b7e3fa7a874c: !Template + 85cf8fa9-af9d-4ade-8113-0022cbb4a295: !Template answer_choices: A ||| B - id: c8cedf93-50df-490d-9f37-b7e3fa7a874c + id: 85cf8fa9-af9d-4ade-8113-0022cbb4a295 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} @@ -158,75 +136,33 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B - reference: '' - cecd6e5a-8daa-4102-9888-4fcd6e7c649f: !Template - answer_choices: Yes ||| No - id: cecd6e5a-8daa-4102-9888-4fcd6e7c649f - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no - reference: '' - e498feb2-f347-4d28-a557-804241a81edf: !Template - answer_choices: Yes ||| No - id: e498feb2-f347-4d28-a557-804241a81edf - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes - reference: '' - e61e6c8a-8231-47e4-8ac0-b989a93f54a9: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: e61e6c8a-8231-47e4-8ac0-b989a93f54a9 - jinja: ' ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: true_null_prompt + name: grammatical_between_A_B reference: '' - f13211a8-ddd8-4aef-9757-4bb6f35fdd30: !Template - answer_choices: 1 ||| 2 - id: f13211a8-ddd8-4aef-9757-4bb6f35fdd30 + a17cf3be-ae52-4cd1-a2a3-348bf74acfa7: !Template + answer_choices: A ||| B + id: a17cf3be-ae52-4cd1-a2a3-348bf74acfa7 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -240,14 +176,37 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: better_A_B reference: '' - f8f8680b-dd66-4b56-a324-ac516008d355: !Template + a48cef96-2d21-4205-aaf8-e46396f3a872: !Template + answer_choices: Yes ||| No + id: a48cef96-2d21-4205-aaf8-e46396f3a872 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + cd6fd211-5af5-4d83-8282-5df007f05a6d: !Template answer_choices: A ||| B - id: f8f8680b-dd66-4b56-a324-ac516008d355 + id: cd6fd211-5af5-4d83-8282-5df007f05a6d jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} @@ -279,43 +238,25 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: grammatical_which_one_A_B reference: '' - fd765908-f30f-46aa-bf93-e2ed4473294b: !Template - answer_choices: A ||| B - id: fd765908-f30f-46aa-bf93-e2ed4473294b + e5ee12bd-b051-4487-8abd-1e0535b1f9cd: !Template + answer_choices: Yes ||| No + id: e5ee12bd-b051-4487-8abd-1e0535b1f9cd jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. - - {% if shuffled_order == 0 %} - - Sentence A: {{ sentence_good }} - - Sentence B: {{ sentence_bad }} - - {% else %} - - Sentence A: {{ sentence_bad }} + {% set shuffled_order_options = [0, 1] | random %} - Sentence B: {{ sentence_good }} - - {% endif %} + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} - - {% endif %}' + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: better_A_B + original_task: false + name: single_sentence_good_yes_no reference: '' diff --git a/promptsource/templates/blimp/determiner_noun_agreement_irregular_2/templates.yaml b/promptsource/templates/blimp/determiner_noun_agreement_irregular_2/templates.yaml index e15bdf756..ee2dbfc88 100644 --- a/promptsource/templates/blimp/determiner_noun_agreement_irregular_2/templates.yaml +++ b/promptsource/templates/blimp/determiner_noun_agreement_irregular_2/templates.yaml @@ -1,10 +1,30 @@ dataset: blimp subset: determiner_noun_agreement_irregular_2 templates: - 06923da0-136b-4bf9-9dc8-d06779e90f0d: !Template + 124e6432-02e3-4e07-9ada-ff24ccd71656: !Template + answer_choices: Yes ||| No + id: 124e6432-02e3-4e07-9ada-ff24ccd71656 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 8406d6c0-2383-4832-b0ff-48ca295944f2: !Template answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 06923da0-136b-4bf9-9dc8-d06779e90f0d - jinja: '" ||| {{ sentence_good }}' + id: 8406d6c0-2383-4832-b0ff-48ca295944f2 + jinja: ' ||| {{ sentence_good }}' metadata: !TemplateMetadata choices_in_prompt: false metrics: @@ -12,28 +32,35 @@ templates: original_task: true name: null_prompt reference: '' - 105467bc-f5bc-41fe-99f8-2ac18bcebc7e: !Template + 86eebca6-aac0-4949-9b00-76ad6fefdda0: !Template answer_choices: Yes ||| No - id: 105467bc-f5bc-41fe-99f8-2ac18bcebc7e - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + id: 86eebca6-aac0-4949-9b00-76ad6fefdda0 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_bad_yes_no + name: single_sentence_good_yes_no reference: '' - 15f33e2a-3e6b-4efe-b71f-89656d3d8f18: !Template + ca5311f3-7066-4e65-8764-64319087ebfa: !Template answer_choices: A ||| B - id: 15f33e2a-3e6b-4efe-b71f-89656d3d8f18 + id: ca5311f3-7066-4e65-8764-64319087ebfa jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} @@ -67,11 +94,13 @@ templates: original_task: true name: grammatical_between_A_B reference: '' - 25c379da-8e98-453e-bfa6-858c0b06c372: !Template + d7f7688c-86d4-4a29-8c4e-e25f016db83b: !Template answer_choices: A ||| B - id: 25c379da-8e98-453e-bfa6-858c0b06c372 + id: d7f7688c-86d4-4a29-8c4e-e25f016db83b jinja: '{% set shuffled_order = [0, 1] | random %} + {% set shuffled_order_options = [0, 1] | random %} + Read the following two sentences. {% if shuffled_order == 0 %} @@ -88,8 +117,8 @@ templates: {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -105,24 +134,27 @@ templates: original_task: true name: better_A_B reference: '' - 7193d72d-63c4-47ce-8722-a3cd69f3d877: !Template - answer_choices: 1 ||| 2 - id: 7193d72d-63c4-47ce-8722-a3cd69f3d877 + e43b8098-ea0f-4786-af61-9bf3f744a7be: !Template + answer_choices: A ||| B + id: e43b8098-ea0f-4786-af61-9bf3f744a7be jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -142,25 +174,17 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 - reference: '' - 76d252bc-44ff-40a8-9b09-bf080461385f: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 76d252bc-44ff-40a8-9b09-bf080461385f - jinja: ' ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: true_null_prompt + name: grammatical_which_one_A_B reference: '' - 8aa80b8c-859f-4a84-9967-83a6e54a30fd: !Template + fc0d9c16-020e-4bd8-91d9-7e56bd56146c: !Template answer_choices: 1 ||| 2 - id: 8aa80b8c-859f-4a84-9967-83a6e54a30fd + id: fc0d9c16-020e-4bd8-91d9-7e56bd56146c jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} @@ -192,42 +216,29 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 - reference: '' - 9d317cfa-cce0-4774-b327-b435838947a1: !Template - answer_choices: Yes ||| No - id: 9d317cfa-cce0-4774-b327-b435838947a1 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes + name: grammatical_which_one_1_2 reference: '' - d1773c16-0d07-451e-b327-915929bc0120: !Template - answer_choices: A ||| B - id: d1773c16-0d07-451e-b327-915929bc0120 + fdfdef44-308e-4b31-9477-0fefe55a20f5: !Template + answer_choices: 1 ||| 2 + id: fdfdef44-308e-4b31-9477-0fefe55a20f5 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -247,75 +258,5 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B - reference: '' - db127d8a-f404-402a-b420-ba9519af2707: !Template - answer_choices: A ||| B - id: db127d8a-f404-402a-b420-ba9519af2707 - jinja: '{% set shuffled_order = [0, 1] | random %} - - Read the following two sentences. - - {% if shuffled_order == 0 %} - - Sentence A: {{ sentence_good }} - - Sentence B: {{ sentence_bad }} - - {% else %} - - Sentence A: {{ sentence_bad }} - - Sentence B: {{ sentence_good }} - - {% endif %} - - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} - - {% endif %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: better_B_A - reference: '' - dbf3fd44-687f-4549-9737-72505fe0067a: !Template - answer_choices: Yes ||| No - id: dbf3fd44-687f-4549-9737-72505fe0067a - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no - reference: '' - e9ecf4cc-8541-4104-b61e-0be797b9a2f8: !Template - answer_choices: Yes ||| No - id: e9ecf4cc-8541-4104-b61e-0be797b9a2f8 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes + name: grammatical_between_1_2 reference: '' diff --git a/promptsource/templates/blimp/determiner_noun_agreement_with_adj_2/templates.yaml b/promptsource/templates/blimp/determiner_noun_agreement_with_adj_2/templates.yaml index 63a4b9181..82c4ccc41 100644 --- a/promptsource/templates/blimp/determiner_noun_agreement_with_adj_2/templates.yaml +++ b/promptsource/templates/blimp/determiner_noun_agreement_with_adj_2/templates.yaml @@ -1,72 +1,33 @@ dataset: blimp subset: determiner_noun_agreement_with_adj_2 templates: - 501487b3-c60b-4164-a3d4-bc19899b570a: !Template - answer_choices: Yes ||| No - id: 501487b3-c60b-4164-a3d4-bc19899b570a - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes - reference: '' - 65bef5ea-aa86-4a1f-bb22-0def0bd2055a: !Template - answer_choices: Yes ||| No - id: 65bef5ea-aa86-4a1f-bb22-0def0bd2055a - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no - reference: '' - 8882dae4-ed47-4d66-ac20-8c4c585abdf0: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 8882dae4-ed47-4d66-ac20-8c4c585abdf0 - jinja: '" ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: null_prompt - reference: '' - ae1ca2f2-4ca2-4d6e-bd32-4167e547df00: !Template - answer_choices: A ||| B - id: ae1ca2f2-4ca2-4d6e-bd32-4167e547df00 + 062fc639-5b95-445b-a161-cb3435c3917e: !Template + answer_choices: 1 ||| 2 + id: 062fc639-5b95-445b-a161-cb3435c3917e jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -80,26 +41,29 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: grammatical_which_one_1_2 reference: '' - b2dbdabb-af0c-42db-afe1-789e9453def5: !Template - answer_choices: 1 ||| 2 - id: b2dbdabb-af0c-42db-afe1-789e9453def5 + 12ebdf0e-4d2e-413a-b356-b20f26d697f9: !Template + answer_choices: A ||| B + id: 12ebdf0e-4d2e-413a-b356-b20f26d697f9 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -119,53 +83,29 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 - reference: '' - bcd4116b-ca3c-4770-b29c-aa3843492b85: !Template - answer_choices: Yes ||| No - id: bcd4116b-ca3c-4770-b29c-aa3843492b85 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes - reference: '' - c073ee55-fe16-4bd6-ba92-2942a73ca136: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: c073ee55-fe16-4bd6-ba92-2942a73ca136 - jinja: ' ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: true_null_prompt + name: grammatical_between_A_B reference: '' - d2917209-4d2b-4468-abe7-2ad11650dcf9: !Template - answer_choices: 1 ||| 2 - id: d2917209-4d2b-4468-abe7-2ad11650dcf9 + 1c886997-c2f6-49fa-8f45-94fe821576da: !Template + answer_choices: A ||| B + id: 1c886997-c2f6-49fa-8f45-94fe821576da jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -185,64 +125,60 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: grammatical_which_one_A_B reference: '' - e5445e04-7d31-47ff-b225-ebae6a991f2c: !Template - answer_choices: A ||| B - id: e5445e04-7d31-47ff-b225-ebae6a991f2c + 207eac77-5fdc-4351-8d58-41a490cddafa: !Template + answer_choices: Yes ||| No + id: 207eac77-5fdc-4351-8d58-41a490cddafa jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. - - {% if shuffled_order == 0 %} - - Sentence A: {{ sentence_good }} - - Sentence B: {{ sentence_bad }} - - {% else %} - - Sentence A: {{ sentence_bad }} - - Sentence B: {{ sentence_good }} - - {% endif %} + {% set shuffled_order_options = [0, 1] | random %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} - - {% endif %}' + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 6cab34a8-4125-4e00-96c9-5adfa41b40cd: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 6cab34a8-4125-4e00-96c9-5adfa41b40cd + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy original_task: true - name: better_A_B + name: null_prompt reference: '' - ed2d4629-29b0-44cb-b459-06aa4c58a66b: !Template - answer_choices: A ||| B - id: ed2d4629-29b0-44cb-b459-06aa4c58a66b + 9cea593b-b7cf-409e-8b4e-3fe1ee52a116: !Template + answer_choices: 1 ||| 2 + id: 9cea593b-b7cf-409e-8b4e-3fe1ee52a116 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -262,48 +198,53 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: grammatical_between_1_2 reference: '' - f3f5612c-61fe-431a-87c1-9092f94780bb: !Template + 9e000ee9-c511-43ff-b3b8-0487681b60f5: !Template answer_choices: Yes ||| No - id: f3f5612c-61fe-431a-87c1-9092f94780bb - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + id: 9e000ee9-c511-43ff-b3b8-0487681b60f5 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_bad_yes_no + name: single_sentence_good_yes_no reference: '' - ffdd944c-ebee-46b9-9434-869197a153ca: !Template + d24e5794-5f01-4bae-bd5d-7a8c83ef33be: !Template answer_choices: A ||| B - id: ffdd944c-ebee-46b9-9434-869197a153ca + id: d24e5794-5f01-4bae-bd5d-7a8c83ef33be jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -317,5 +258,5 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: better_A_B reference: '' diff --git a/promptsource/templates/blimp/determiner_noun_agreement_with_adj_irregular_1/templates.yaml b/promptsource/templates/blimp/determiner_noun_agreement_with_adj_irregular_1/templates.yaml index cc973cb3c..d6f0af25a 100644 --- a/promptsource/templates/blimp/determiner_noun_agreement_with_adj_irregular_1/templates.yaml +++ b/promptsource/templates/blimp/determiner_noun_agreement_with_adj_irregular_1/templates.yaml @@ -1,30 +1,31 @@ dataset: blimp subset: determiner_noun_agreement_with_adj_irregular_1 templates: - 063a3b4f-5603-4976-99ef-30f335a92a4c: !Template - answer_choices: 1 ||| 2 - id: 063a3b4f-5603-4976-99ef-30f335a92a4c + 2a96a2a0-b391-44aa-991f-a1dd3daa2c83: !Template + answer_choices: A ||| B + id: 2a96a2a0-b391-44aa-991f-a1dd3daa2c83 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -38,57 +39,17 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 - reference: '' - 0c7a8539-45d5-4281-8650-b26ab0b84e9f: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 0c7a8539-45d5-4281-8650-b26ab0b84e9f - jinja: ' ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: true_null_prompt - reference: '' - 11b62273-ecf0-4677-8a33-21800a7bdecb: !Template - answer_choices: Yes ||| No - id: 11b62273-ecf0-4677-8a33-21800a7bdecb - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes - reference: '' - 1dfe5621-892e-4708-9a07-575d48b7cfb3: !Template - answer_choices: Yes ||| No - id: 1dfe5621-892e-4708-9a07-575d48b7cfb3 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no + name: better_A_B reference: '' - 230f386f-75fe-47c6-9790-dad308ebb798: !Template + 2c819b61-7921-4827-a02c-c269cb857432: !Template answer_choices: 1 ||| 2 - id: 230f386f-75fe-47c6-9790-dad308ebb798 + id: 2c819b61-7921-4827-a02c-c269cb857432 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} @@ -122,62 +83,67 @@ templates: original_task: true name: grammatical_which_one_1_2 reference: '' - 2cbf40f2-612d-4f91-a3b3-52867c1e3be8: !Template - answer_choices: A ||| B - id: 2cbf40f2-612d-4f91-a3b3-52867c1e3be8 + 3c996321-603c-4d6f-a536-784916f9ca54: !Template + answer_choices: Yes ||| No + id: 3c996321-603c-4d6f-a536-784916f9ca54 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + {% set shuffled_order_options = [0, 1] | random %} - {% if shuffled_order == 0 %} + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" - Sentence A: {{ sentence_good }} - - Sentence B: {{ sentence_bad }} - - {% else %} - - Sentence A: {{ sentence_bad }} - - Sentence B: {{ sentence_good }} - - {% endif %} - - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} - - {{ answer_choices[0] }} + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 55a34363-0c99-4f77-94b3-72055c9deff0: !Template + answer_choices: Yes ||| No + id: 55a34363-0c99-4f77-94b3-72055c9deff0 + jinja: '{% set shuffled_order = [0, 1] | random %} - {% else %} + {% set shuffled_order_options = [0, 1] | random %} - {{ answer_choices[1] }} + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" - {% endif %}' + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: better_A_B + original_task: false + name: single_sentence_bad_yes_no reference: '' - 42417d55-e14b-4fe6-ad46-133a1d93d43f: !Template - answer_choices: A ||| B - id: 42417d55-e14b-4fe6-ad46-133a1d93d43f + 782fff30-473e-492f-9570-edacf7b52b06: !Template + answer_choices: 1 ||| 2 + id: 782fff30-473e-492f-9570-edacf7b52b06 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -197,31 +163,35 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: grammatical_between_1_2 reference: '' - 56380f29-db99-416d-92d5-044298d030bf: !Template + 8257567f-5280-4e6a-89ac-4aed01c1befb: !Template answer_choices: A ||| B - id: 56380f29-db99-416d-92d5-044298d030bf + id: 8257567f-5280-4e6a-89ac-4aed01c1befb jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -235,14 +205,17 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: grammatical_which_one_A_B reference: '' - a0d63188-a913-4ad9-bbc8-3b315d23dee1: !Template + c8555faf-50e2-4785-ad8a-41bf325016a6: !Template answer_choices: A ||| B - id: a0d63188-a913-4ad9-bbc8-3b315d23dee1 + id: c8555faf-50e2-4785-ad8a-41bf325016a6 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} @@ -276,42 +249,10 @@ templates: original_task: true name: grammatical_between_A_B reference: '' - acd86c1a-5fb1-4581-b455-231d325de841: !Template - answer_choices: Yes ||| No - id: acd86c1a-5fb1-4581-b455-231d325de841 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no - reference: '' - c70028f7-f29a-4d7f-bd0a-eb584855d7b4: !Template - answer_choices: Yes ||| No - id: c70028f7-f29a-4d7f-bd0a-eb584855d7b4 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes - reference: '' - ed1d67ab-0a04-4634-ac00-faa5eaabde4b: !Template + f2b79942-dd9f-40a6-8ef5-96e8b519b08f: !Template answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: ed1d67ab-0a04-4634-ac00-faa5eaabde4b - jinja: '" ||| {{ sentence_good }}' + id: f2b79942-dd9f-40a6-8ef5-96e8b519b08f + jinja: ' ||| {{ sentence_good }}' metadata: !TemplateMetadata choices_in_prompt: false metrics: diff --git a/promptsource/templates/blimp/determiner_noun_agreement_with_adj_irregular_2/templates.yaml b/promptsource/templates/blimp/determiner_noun_agreement_with_adj_irregular_2/templates.yaml index 215cc41bd..a417782e1 100644 --- a/promptsource/templates/blimp/determiner_noun_agreement_with_adj_irregular_2/templates.yaml +++ b/promptsource/templates/blimp/determiner_noun_agreement_with_adj_irregular_2/templates.yaml @@ -1,40 +1,27 @@ dataset: blimp subset: determiner_noun_agreement_with_adj_irregular_2 templates: - 0cb0767a-40f3-4ff0-aebe-e1a5c33c976d: !Template - answer_choices: Yes ||| No - id: 0cb0767a-40f3-4ff0-aebe-e1a5c33c976d - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes - reference: '' - 20257e24-2ef9-419b-a74d-b96cef4fd3e1: !Template - answer_choices: 1 ||| 2 - id: 20257e24-2ef9-419b-a74d-b96cef4fd3e1 + 05972c48-b382-43ba-9afb-8664220284b4: !Template + answer_choices: A ||| B + id: 05972c48-b382-43ba-9afb-8664220284b4 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -54,13 +41,35 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: grammatical_which_one_A_B + reference: '' + 143373b3-b7c7-42f8-924f-b7e308218c4c: !Template + answer_choices: Yes ||| No + id: 143373b3-b7c7-42f8-924f-b7e308218c4c + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no reference: '' - 35ef4d06-1b5d-4191-8599-df70048bc208: !Template + 30faf1bc-e1ba-492a-b07a-a8e6eb06d248: !Template answer_choices: A ||| B - id: 35ef4d06-1b5d-4191-8599-df70048bc208 + id: 30faf1bc-e1ba-492a-b07a-a8e6eb06d248 jinja: '{% set shuffled_order = [0, 1] | random %} + {% set shuffled_order_options = [0, 1] | random %} + Read the following two sentences. {% if shuffled_order == 0 %} @@ -77,8 +86,8 @@ templates: {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -94,12 +103,15 @@ templates: original_task: true name: better_A_B reference: '' - 4297cd21-25af-4c96-98ab-760591b3bff4: !Template + 509b6572-71f2-49a7-ab5b-bf1a3bb869a8: !Template answer_choices: 1 ||| 2 - id: 4297cd21-25af-4c96-98ab-760591b3bff4 + id: 509b6572-71f2-49a7-ab5b-bf1a3bb869a8 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} @@ -133,65 +145,10 @@ templates: original_task: true name: grammatical_between_1_2 reference: '' - 5b520f6e-b2da-47bd-a01c-1d6b0d7b8d84: !Template - answer_choices: A ||| B - id: 5b520f6e-b2da-47bd-a01c-1d6b0d7b8d84 - jinja: '{% set shuffled_order = [0, 1] | random %} - - Which one of the following sentences is grammatical? Please answer A or B. - - {% if shuffled_order == 0 %} - - A: {{ sentence_good }} - - B: {{ sentence_bad }} - - {% else %} - - A: {{ sentence_bad }} - - B: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} - - {% endif %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: grammatical_which_one_A_B - reference: '' - 7baeee4a-29ee-4c12-b205-a7348c8ea042: !Template - answer_choices: Yes ||| No - id: 7baeee4a-29ee-4c12-b205-a7348c8ea042 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes - reference: '' - 7de33be7-58d4-4760-b905-1f48db075936: !Template + 6630731c-8593-4471-a0d1-b3547a176bdb: !Template answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 7de33be7-58d4-4760-b905-1f48db075936 - jinja: '" ||| {{ sentence_good }}' + id: 6630731c-8593-4471-a0d1-b3547a176bdb + jinja: ' ||| {{ sentence_good }}' metadata: !TemplateMetadata choices_in_prompt: false metrics: @@ -199,28 +156,15 @@ templates: original_task: true name: null_prompt reference: '' - 8f07a059-1366-44a0-b728-620535232ac3: !Template - answer_choices: Yes ||| No - id: 8f07a059-1366-44a0-b728-620535232ac3 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no - reference: '' - 96eb1910-db1c-4917-9f17-3b7be652aa27: !Template + 8f65d37e-8e47-429b-b56d-c6762842663b: !Template answer_choices: A ||| B - id: 96eb1910-db1c-4917-9f17-3b7be652aa27 + id: 8f65d37e-8e47-429b-b56d-c6762842663b jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} @@ -254,40 +198,53 @@ templates: original_task: true name: grammatical_between_A_B reference: '' - b8885a4a-7fac-4208-8f75-02a92e86c7db: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: b8885a4a-7fac-4208-8f75-02a92e86c7db - jinja: ' ||| {{ sentence_good }}' + d7a8ee44-8bb6-4575-b1e0-b1b36461a0fa: !Template + answer_choices: Yes ||| No + id: d7a8ee44-8bb6-4575-b1e0-b1b36461a0fa + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata - choices_in_prompt: false + choices_in_prompt: true metrics: - Accuracy - original_task: true - name: true_null_prompt + original_task: false + name: single_sentence_good_yes_no reference: '' - ee080fdb-8b82-4226-98c2-793a39b13d64: !Template - answer_choices: A ||| B - id: ee080fdb-8b82-4226-98c2-793a39b13d64 + f9ae1c45-9f80-4c9b-88fb-6969c72cc5ed: !Template + answer_choices: 1 ||| 2 + id: f9ae1c45-9f80-4c9b-88fb-6969c72cc5ed jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -301,21 +258,5 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A - reference: '' - f3a2b574-0c67-4d66-9772-54b13ecafc9b: !Template - answer_choices: Yes ||| No - id: f3a2b574-0c67-4d66-9772-54b13ecafc9b - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no + name: grammatical_which_one_1_2 reference: '' diff --git a/promptsource/templates/blimp/determiner_noun_agreement_with_adjective_1/templates.yaml b/promptsource/templates/blimp/determiner_noun_agreement_with_adjective_1/templates.yaml index 438514cc4..679e608c2 100644 --- a/promptsource/templates/blimp/determiner_noun_agreement_with_adjective_1/templates.yaml +++ b/promptsource/templates/blimp/determiner_noun_agreement_with_adjective_1/templates.yaml @@ -1,39 +1,15 @@ dataset: blimp subset: determiner_noun_agreement_with_adjective_1 templates: - 452a4bff-9c60-49b3-85b4-8ae6267fb36b: !Template - answer_choices: Yes ||| No - id: 452a4bff-9c60-49b3-85b4-8ae6267fb36b - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no - reference: '' - 5072949c-2ceb-48e5-be2c-d0175ac71207: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 5072949c-2ceb-48e5-be2c-d0175ac71207 - jinja: '" ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: null_prompt - reference: '' - 517dda30-4692-43f0-b448-ba30836314e0: !Template + 16fc0d72-7188-400a-9373-6612ed85dca4: !Template answer_choices: A ||| B - id: 517dda30-4692-43f0-b448-ba30836314e0 + id: 16fc0d72-7188-400a-9373-6612ed85dca4 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} @@ -67,45 +43,33 @@ templates: original_task: true name: grammatical_between_A_B reference: '' - 593639c9-5d3e-4cce-9758-be2003dbf8cc: !Template - answer_choices: Yes ||| No - id: 593639c9-5d3e-4cce-9758-be2003dbf8cc - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no - reference: '' - 5b43a31c-1173-485a-820b-71087d79b0bc: !Template - answer_choices: A ||| B - id: 5b43a31c-1173-485a-820b-71087d79b0bc + 3403d371-ebcf-4212-b1ca-1d392f7e9907: !Template + answer_choices: 1 ||| 2 + id: 3403d371-ebcf-4212-b1ca-1d392f7e9907 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -119,32 +83,33 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: grammatical_which_one_1_2 reference: '' - 6a4dd847-92d0-4bdc-88ac-62d8b3a8aa31: !Template + 3ee681df-24b4-43f8-a9de-39f5c8614d7d: !Template answer_choices: A ||| B - id: 6a4dd847-92d0-4bdc-88ac-62d8b3a8aa31 + id: 3ee681df-24b4-43f8-a9de-39f5c8614d7d jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -158,53 +123,60 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B - reference: '' - 79fb5615-290d-4435-8d1a-bda7afd69e9b: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 79fb5615-290d-4435-8d1a-bda7afd69e9b - jinja: ' ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: true_null_prompt + name: better_A_B reference: '' - b03e0373-24f7-4cf3-9de0-ce770c7c3e2f: !Template + 48036274-c1c3-4e30-8fcc-bb159797cbbf: !Template answer_choices: Yes ||| No - id: b03e0373-24f7-4cf3-9de0-ce770c7c3e2f - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + id: 48036274-c1c3-4e30-8fcc-bb159797cbbf + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_bad_no_yes + name: single_sentence_bad_yes_no reference: '' - c5191b94-65b9-4562-9438-84bfb78bd4c8: !Template - answer_choices: 1 ||| 2 - id: c5191b94-65b9-4562-9438-84bfb78bd4c8 + 8261a763-d444-46a2-a223-891370d55245: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 8261a763-d444-46a2-a223-891370d55245 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + 87eeacd9-a39a-4dc3-9814-85410875666a: !Template + answer_choices: A ||| B + id: 87eeacd9-a39a-4dc3-9814-85410875666a jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -224,52 +196,37 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: grammatical_which_one_A_B reference: '' - e0d4bab2-7fd4-4b4b-a378-beaafd9f9e49: !Template - answer_choices: A ||| B - id: e0d4bab2-7fd4-4b4b-a378-beaafd9f9e49 + a0eba446-b21c-41ff-bbbc-e55c40bcebbf: !Template + answer_choices: Yes ||| No + id: a0eba446-b21c-41ff-bbbc-e55c40bcebbf jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. - - {% if shuffled_order == 0 %} + {% set shuffled_order_options = [0, 1] | random %} - Sentence A: {{ sentence_good }} + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" - Sentence B: {{ sentence_bad }} - - {% else %} - - Sentence A: {{ sentence_bad }} - - Sentence B: {{ sentence_good }} - - {% endif %} - - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} - - {% endif %}' + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: better_B_A + original_task: false + name: single_sentence_good_yes_no reference: '' - e1ec22cd-feb1-404f-8124-98e0874533a4: !Template + fd1cc83f-521c-4b3d-8b86-fe0ce9e4cbcf: !Template answer_choices: 1 ||| 2 - id: e1ec22cd-feb1-404f-8124-98e0874533a4 + id: fd1cc83f-521c-4b3d-8b86-fe0ce9e4cbcf jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} @@ -301,21 +258,5 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 - reference: '' - f17e6002-bd89-4c21-b631-0cb20faa5912: !Template - answer_choices: Yes ||| No - id: f17e6002-bd89-4c21-b631-0cb20faa5912 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes + name: grammatical_between_1_2 reference: '' diff --git a/promptsource/templates/blimp/distractor_agreement_relational_noun/templates.yaml b/promptsource/templates/blimp/distractor_agreement_relational_noun/templates.yaml index 8309459d6..49b28e325 100644 --- a/promptsource/templates/blimp/distractor_agreement_relational_noun/templates.yaml +++ b/promptsource/templates/blimp/distractor_agreement_relational_noun/templates.yaml @@ -1,24 +1,27 @@ dataset: blimp subset: distractor_agreement_relational_noun templates: - 028695a2-4f72-4aae-af65-0f6373512a76: !Template - answer_choices: 1 ||| 2 - id: 028695a2-4f72-4aae-af65-0f6373512a76 + 04e6bf51-c73a-4089-986b-789729e3db62: !Template + answer_choices: A ||| B + id: 04e6bf51-c73a-4089-986b-789729e3db62 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -38,31 +41,35 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: grammatical_between_A_B reference: '' - 2024f0d9-491c-46fd-b8f5-53e34db8853d: !Template - answer_choices: A ||| B - id: 2024f0d9-491c-46fd-b8f5-53e34db8853d + 37b91041-3225-4291-baa3-33a85d3c6071: !Template + answer_choices: 1 ||| 2 + id: 37b91041-3225-4291-baa3-33a85d3c6071 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -76,32 +83,53 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: grammatical_between_1_2 + reference: '' + 70527b1b-38e3-4e50-9142-3a6a4b068d35: !Template + answer_choices: Yes ||| No + id: 70527b1b-38e3-4e50-9142-3a6a4b068d35 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no reference: '' - 2c62cdb9-adc7-458c-9b28-1120c1f98b7f: !Template + 736d891c-e000-4d0f-b2df-b343c501ad2f: !Template answer_choices: A ||| B - id: 2c62cdb9-adc7-458c-9b28-1120c1f98b7f + id: 736d891c-e000-4d0f-b2df-b343c501ad2f jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -115,14 +143,17 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: better_A_B reference: '' - 3b5ade02-1f86-48a9-b194-71c3c5033fcd: !Template + dfb67083-db29-415c-bdc8-6f25b97a1293: !Template answer_choices: A ||| B - id: 3b5ade02-1f86-48a9-b194-71c3c5033fcd + id: dfb67083-db29-415c-bdc8-6f25b97a1293 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} @@ -156,15 +187,19 @@ templates: original_task: true name: grammatical_which_one_A_B reference: '' - 61fb2ae7-ea43-4f13-9e72-9f05e98668d6: !Template + e0e9dadb-9638-4b87-a0f4-666e5c17b94e: !Template answer_choices: Yes ||| No - id: 61fb2ae7-ea43-4f13-9e72-9f05e98668d6 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + id: e0e9dadb-9638-4b87-a0f4-666e5c17b94e + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: @@ -172,12 +207,15 @@ templates: original_task: false name: single_sentence_good_yes_no reference: '' - 7572752b-cf67-4e8d-94d9-eb85b87c7ad1: !Template + e7e10ac3-980a-4d0f-874e-72210e1f42cf: !Template answer_choices: 1 ||| 2 - id: 7572752b-cf67-4e8d-94d9-eb85b87c7ad1 + id: e7e10ac3-980a-4d0f-874e-72210e1f42cf jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} @@ -209,113 +247,16 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 - reference: '' - 795af14f-7b5b-402a-bb76-c5db559618a7: !Template - answer_choices: Yes ||| No - id: 795af14f-7b5b-402a-bb76-c5db559618a7 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes - reference: '' - 8f7a259e-840f-441f-9746-6061c5b7658b: !Template - answer_choices: Yes ||| No - id: 8f7a259e-840f-441f-9746-6061c5b7658b - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes - reference: '' - 97afeded-6b3d-4e53-91ba-b4dccf126e83: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 97afeded-6b3d-4e53-91ba-b4dccf126e83 - jinja: '" ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: null_prompt - reference: '' - a0b01db7-8fe8-4720-939b-bf71f35d370c: !Template - answer_choices: A ||| B - id: a0b01db7-8fe8-4720-939b-bf71f35d370c - jinja: '{% set shuffled_order = [0, 1] | random %} - - Read the following two sentences. - - {% if shuffled_order == 0 %} - - Sentence A: {{ sentence_good }} - - Sentence B: {{ sentence_bad }} - - {% else %} - - Sentence A: {{ sentence_bad }} - - Sentence B: {{ sentence_good }} - - {% endif %} - - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} - - {% endif %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: better_B_A - reference: '' - a19d24a0-c9d0-46ba-a1dc-eef505ab211a: !Template - answer_choices: Yes ||| No - id: a19d24a0-c9d0-46ba-a1dc-eef505ab211a - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no + name: grammatical_which_one_1_2 reference: '' - f54a1fe1-20a6-44fc-b3de-245d8500e36f: !Template + edc1ffa1-ef01-418f-b5e7-14f2d5f0f801: !Template answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: f54a1fe1-20a6-44fc-b3de-245d8500e36f + id: edc1ffa1-ef01-418f-b5e7-14f2d5f0f801 jinja: ' ||| {{ sentence_good }}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - Accuracy original_task: true - name: true_null_prompt + name: null_prompt reference: '' diff --git a/promptsource/templates/blimp/distractor_agreement_relative_clause/templates.yaml b/promptsource/templates/blimp/distractor_agreement_relative_clause/templates.yaml index 9591fd05d..ece833081 100644 --- a/promptsource/templates/blimp/distractor_agreement_relative_clause/templates.yaml +++ b/promptsource/templates/blimp/distractor_agreement_relative_clause/templates.yaml @@ -1,10 +1,10 @@ dataset: blimp subset: distractor_agreement_relative_clause templates: - 02aa781c-0c1a-4200-91a4-3f6976461a27: !Template + 031e9719-1408-4a21-8fdf-3bad71fbb2fc: !Template answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 02aa781c-0c1a-4200-91a4-3f6976461a27 - jinja: '" ||| {{ sentence_good }}' + id: 031e9719-1408-4a21-8fdf-3bad71fbb2fc + jinja: ' ||| {{ sentence_good }}' metadata: !TemplateMetadata choices_in_prompt: false metrics: @@ -12,28 +12,15 @@ templates: original_task: true name: null_prompt reference: '' - 0db6fa0a-e41c-4c0e-9fd6-03e86cb7f546: !Template - answer_choices: Yes ||| No - id: 0db6fa0a-e41c-4c0e-9fd6-03e86cb7f546 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes - reference: '' - 126a54c7-4ccd-4114-be6d-b21fca2f6bb6: !Template + 343cf095-39b4-4e4e-912b-84b116217991: !Template answer_choices: 1 ||| 2 - id: 126a54c7-4ccd-4114-be6d-b21fca2f6bb6 + id: 343cf095-39b4-4e4e-912b-84b116217991 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} @@ -67,12 +54,15 @@ templates: original_task: true name: grammatical_between_1_2 reference: '' - 16b1cd6b-048a-40a5-af93-32b5f4eb2820: !Template + 38c20a09-df68-46e7-9f75-7e08474fb3d9: !Template answer_choices: A ||| B - id: 16b1cd6b-048a-40a5-af93-32b5f4eb2820 + id: 38c20a09-df68-46e7-9f75-7e08474fb3d9 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} @@ -106,67 +96,27 @@ templates: original_task: true name: grammatical_between_A_B reference: '' - 59f00aca-ba2e-4e7e-a73e-9571cc819313: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 59f00aca-ba2e-4e7e-a73e-9571cc819313 - jinja: ' ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: true_null_prompt - reference: '' - 620c3f0b-ebec-45c0-bcd2-9c8d43471f41: !Template - answer_choices: Yes ||| No - id: 620c3f0b-ebec-45c0-bcd2-9c8d43471f41 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes - reference: '' - 9315385b-5010-468a-9c4d-dd343bc0148b: !Template - answer_choices: Yes ||| No - id: 9315385b-5010-468a-9c4d-dd343bc0148b - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no - reference: '' - a5d590e5-389e-4244-a460-fc301d2b4866: !Template - answer_choices: A ||| B - id: a5d590e5-389e-4244-a460-fc301d2b4866 + 6a81bb6e-2c7e-496e-80e0-fc49587f9334: !Template + answer_choices: 1 ||| 2 + id: 6a81bb6e-2c7e-496e-80e0-fc49587f9334 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -186,55 +136,41 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: grammatical_which_one_1_2 reference: '' - b09975e3-7e3e-42e5-a623-985ce049eda2: !Template - answer_choices: A ||| B - id: b09975e3-7e3e-42e5-a623-985ce049eda2 + 8256789f-0e85-4bd9-b019-98fcabdeaa77: !Template + answer_choices: Yes ||| No + id: 8256789f-0e85-4bd9-b019-98fcabdeaa77 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + {% set shuffled_order_options = [0, 1] | random %} - {% if shuffled_order == 0 %} + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" - Sentence A: {{ sentence_good }} - - Sentence B: {{ sentence_bad }} - - {% else %} - - Sentence A: {{ sentence_bad }} - - Sentence B: {{ sentence_good }} - - {% endif %} - - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} - - {% endif %}' + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: better_A_B + original_task: false + name: single_sentence_good_yes_no reference: '' - d3a10065-5921-4bdf-944a-b0d062911df0: !Template + 8c8ee811-a4b0-4822-8c03-f53016f87554: !Template answer_choices: Yes ||| No - id: d3a10065-5921-4bdf-944a-b0d062911df0 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + id: 8c8ee811-a4b0-4822-8c03-f53016f87554 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: @@ -242,30 +178,31 @@ templates: original_task: false name: single_sentence_bad_yes_no reference: '' - e62391a2-d78b-4d00-b821-0340e39b75b1: !Template - answer_choices: 1 ||| 2 - id: e62391a2-d78b-4d00-b821-0340e39b75b1 + 99f31214-16a8-4934-895d-30e0f3551426: !Template + answer_choices: A ||| B + id: 99f31214-16a8-4934-895d-30e0f3551426 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -279,31 +216,35 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: better_A_B reference: '' - fea7b9d3-46af-423f-aad8-8db1653fb086: !Template + ff14c20d-654c-4eca-b397-9f873d8df853: !Template answer_choices: A ||| B - id: fea7b9d3-46af-423f-aad8-8db1653fb086 + id: ff14c20d-654c-4eca-b397-9f873d8df853 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -317,5 +258,5 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: grammatical_which_one_A_B reference: '' diff --git a/promptsource/templates/blimp/drop_argument/templates.yaml b/promptsource/templates/blimp/drop_argument/templates.yaml index 50c3b2517..39bb80d3f 100644 --- a/promptsource/templates/blimp/drop_argument/templates.yaml +++ b/promptsource/templates/blimp/drop_argument/templates.yaml @@ -1,83 +1,47 @@ dataset: blimp subset: drop_argument templates: - 0141184b-09b2-4d5a-9f76-9d4db8612e65: !Template + 2d421675-b645-4f6c-b166-a3f79b16be9b: !Template answer_choices: Yes ||| No - id: 0141184b-09b2-4d5a-9f76-9d4db8612e65 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + id: 2d421675-b645-4f6c-b166-a3f79b16be9b + jinja: '{% set shuffled_order = [0, 1] | random %} - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes - reference: '' - 1441372d-f77d-46b4-b185-89b8855282ca: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 1441372d-f77d-46b4-b185-89b8855282ca - jinja: '" ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: null_prompt - reference: '' - 3524a6ac-4e76-4252-b0fd-91f6be66366d: !Template - answer_choices: Yes ||| No - id: 3524a6ac-4e76-4252-b0fd-91f6be66366d - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + {% set shuffled_order_options = [0, 1] | random %} - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no - reference: '' - 3885563f-e59b-4e3e-b086-3312c4ce2544: !Template - answer_choices: Yes ||| No - id: 3885563f-e59b-4e3e-b086-3312c4ce2544 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_good_no_yes + name: single_sentence_good_yes_no reference: '' - 394a377f-2e86-4aa5-8e13-10223b9e1a18: !Template - answer_choices: 1 ||| 2 - id: 394a377f-2e86-4aa5-8e13-10223b9e1a18 + 4c353c08-d69b-48cd-8003-17f3e03da2f3: !Template + answer_choices: A ||| B + id: 4c353c08-d69b-48cd-8003-17f3e03da2f3 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -97,47 +61,46 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: grammatical_between_A_B reference: '' - 6e1f6c6e-1bc2-46fb-9cf4-bdca420c20ac: !Template - answer_choices: Yes ||| No - id: 6e1f6c6e-1bc2-46fb-9cf4-bdca420c20ac - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + 53cc7b68-cf20-44f0-ba0a-c12289e82e95: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 53cc7b68-cf20-44f0-ba0a-c12289e82e95 + jinja: ' ||| {{ sentence_good }}' metadata: !TemplateMetadata - choices_in_prompt: true + choices_in_prompt: false metrics: - Accuracy - original_task: false - name: single_sentence_good_yes_no + original_task: true + name: null_prompt reference: '' - 9f63829b-12e5-4ad5-95f0-0ac57d005567: !Template - answer_choices: A ||| B - id: 9f63829b-12e5-4ad5-95f0-0ac57d005567 + 854e5efa-794f-48dd-a3fe-7027064af3ee: !Template + answer_choices: 1 ||| 2 + id: 854e5efa-794f-48dd-a3fe-7027064af3ee jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -151,76 +114,49 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B - reference: '' - 9fabf622-f791-408e-be16-fa47fae7134e: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 9fabf622-f791-408e-be16-fa47fae7134e - jinja: ' ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: true_null_prompt + name: grammatical_which_one_1_2 reference: '' - a769e479-75dc-4073-9d17-ab4684040ee7: !Template - answer_choices: A ||| B - id: a769e479-75dc-4073-9d17-ab4684040ee7 + 9398d73d-00a0-48f2-b667-e2ea8ab743ec: !Template + answer_choices: Yes ||| No + id: 9398d73d-00a0-48f2-b667-e2ea8ab743ec jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} - {% if shuffled_order == 0 %} - - A: {{ sentence_good }} - - B: {{ sentence_bad }} - - {% else %} + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" - A: {{ sentence_bad }} - - B: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} - - {% endif %}' + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: grammatical_which_one_A_B + original_task: false + name: single_sentence_bad_yes_no reference: '' - d4e1cb9c-1402-4583-8103-6708b39354c8: !Template - answer_choices: A ||| B - id: d4e1cb9c-1402-4583-8103-6708b39354c8 + 9f31c997-9c83-4ee4-a879-354047af5756: !Template + answer_choices: 1 ||| 2 + id: 9f31c997-9c83-4ee4-a879-354047af5756 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -240,13 +176,15 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: grammatical_between_1_2 reference: '' - e08a3cbf-3938-4409-9a76-e8d46590c5e2: !Template + cb6ac991-e5a3-4325-beca-fb4ec479ce45: !Template answer_choices: A ||| B - id: e08a3cbf-3938-4409-9a76-e8d46590c5e2 + id: cb6ac991-e5a3-4325-beca-fb4ec479ce45 jinja: '{% set shuffled_order = [0, 1] | random %} + {% set shuffled_order_options = [0, 1] | random %} + Read the following two sentences. {% if shuffled_order == 0 %} @@ -263,8 +201,8 @@ templates: {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -278,26 +216,29 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: better_A_B reference: '' - e701cdc9-f85b-4bee-99f9-caaf18de060c: !Template - answer_choices: 1 ||| 2 - id: e701cdc9-f85b-4bee-99f9-caaf18de060c + f2204727-5f8c-491f-aff8-317fb4d87e1d: !Template + answer_choices: A ||| B + id: f2204727-5f8c-491f-aff8-317fb4d87e1d jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -317,5 +258,5 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: grammatical_which_one_A_B reference: '' diff --git a/promptsource/templates/blimp/ellipsis_n_bar_1/templates.yaml b/promptsource/templates/blimp/ellipsis_n_bar_1/templates.yaml index 978816f61..e47ceaebc 100644 --- a/promptsource/templates/blimp/ellipsis_n_bar_1/templates.yaml +++ b/promptsource/templates/blimp/ellipsis_n_bar_1/templates.yaml @@ -1,31 +1,39 @@ dataset: blimp subset: ellipsis_n_bar_1 templates: - 07e32c6c-e950-4e8a-b401-33b11a70f31c: !Template + 059b8cb5-6372-4986-9015-e26c43a1294f: !Template answer_choices: Yes ||| No - id: 07e32c6c-e950-4e8a-b401-33b11a70f31c - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + id: 059b8cb5-6372-4986-9015-e26c43a1294f + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_good_no_yes + name: single_sentence_good_yes_no reference: '' - 3f44acfa-cb27-4366-aad6-610556aee656: !Template + 11f3090a-0a82-40be-a007-78f492e4ae97: !Template answer_choices: Yes ||| No - id: 3f44acfa-cb27-4366-aad6-610556aee656 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + id: 11f3090a-0a82-40be-a007-78f492e4ae97 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: @@ -33,62 +41,27 @@ templates: original_task: false name: single_sentence_bad_yes_no reference: '' - 59745b65-3cc9-4634-832e-448f9dc1f1ac: !Template + 1afe2ffd-1348-49bf-a893-9558832c2276: !Template answer_choices: A ||| B - id: 59745b65-3cc9-4634-832e-448f9dc1f1ac + id: 1afe2ffd-1348-49bf-a893-9558832c2276 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. - - {% if shuffled_order == 0 %} - - Sentence A: {{ sentence_good }} - - Sentence B: {{ sentence_bad }} - - {% else %} - - Sentence A: {{ sentence_bad }} - - Sentence B: {{ sentence_good }} - - {% endif %} - - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} - - {% endif %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: better_B_A - reference: '' - 7e88d1d7-fdd7-4266-8779-4027fbfbd85c: !Template - answer_choices: 1 ||| 2 - id: 7e88d1d7-fdd7-4266-8779-4027fbfbd85c - jinja: '{% set shuffled_order = [0, 1] | random %} + {% set shuffled_order_options = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -108,40 +81,15 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 - reference: '' - 8d6391b8-b14c-4568-b69d-9338315a473e: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 8d6391b8-b14c-4568-b69d-9338315a473e - jinja: '" ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: null_prompt - reference: '' - b0fc4064-25be-444b-81b5-ab99165f5439: !Template - answer_choices: Yes ||| No - id: b0fc4064-25be-444b-81b5-ab99165f5439 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes + name: grammatical_between_A_B reference: '' - b18e6caa-ed61-4af3-91ce-225e7f4a1e10: !Template + 1d77964b-9969-4312-be69-9c628c2adaee: !Template answer_choices: A ||| B - id: b18e6caa-ed61-4af3-91ce-225e7f4a1e10 + id: 1d77964b-9969-4312-be69-9c628c2adaee jinja: '{% set shuffled_order = [0, 1] | random %} + {% set shuffled_order_options = [0, 1] | random %} + Read the following two sentences. {% if shuffled_order == 0 %} @@ -158,8 +106,8 @@ templates: {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -175,24 +123,38 @@ templates: original_task: true name: better_A_B reference: '' - b99af5e1-1fe2-4704-9c41-cb77ba55a638: !Template - answer_choices: A ||| B - id: b99af5e1-1fe2-4704-9c41-cb77ba55a638 + 66090200-488a-4802-95ff-05c6aba62170: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 66090200-488a-4802-95ff-05c6aba62170 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + 999aa38b-d71f-4610-99c8-238bb7dde54a: !Template + answer_choices: 1 ||| 2 + id: 999aa38b-d71f-4610-99c8-238bb7dde54a jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -212,25 +174,17 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B - reference: '' - ba51725d-70ee-4bd0-8337-c31cc6b43408: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: ba51725d-70ee-4bd0-8337-c31cc6b43408 - jinja: ' ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: true_null_prompt + name: grammatical_between_1_2 reference: '' - c8e1309f-4549-4412-8ffd-b2a42b4fd0ac: !Template + d5fad176-7509-421a-b8ee-757976985f3d: !Template answer_choices: 1 ||| 2 - id: c8e1309f-4549-4412-8ffd-b2a42b4fd0ac + id: d5fad176-7509-421a-b8ee-757976985f3d jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} @@ -264,12 +218,15 @@ templates: original_task: true name: grammatical_which_one_1_2 reference: '' - ceaf2e49-0733-4905-908b-04278cd625da: !Template + eab1c212-810a-4a09-94a3-d7bf35697910: !Template answer_choices: A ||| B - id: ceaf2e49-0733-4905-908b-04278cd625da + id: eab1c212-810a-4a09-94a3-d7bf35697910 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} @@ -303,19 +260,3 @@ templates: original_task: true name: grammatical_which_one_A_B reference: '' - f4e414fc-15c3-4f69-9d23-9ba74bb3bef3: !Template - answer_choices: Yes ||| No - id: f4e414fc-15c3-4f69-9d23-9ba74bb3bef3 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no - reference: '' diff --git a/promptsource/templates/blimp/ellipsis_n_bar_2/templates.yaml b/promptsource/templates/blimp/ellipsis_n_bar_2/templates.yaml index 5b6bdcc3d..7fd3f5f99 100644 --- a/promptsource/templates/blimp/ellipsis_n_bar_2/templates.yaml +++ b/promptsource/templates/blimp/ellipsis_n_bar_2/templates.yaml @@ -1,29 +1,33 @@ dataset: blimp subset: ellipsis_n_bar_2 templates: - 00b415b3-ad67-447c-bbc1-dcce2eba458e: !Template - answer_choices: A ||| B - id: 00b415b3-ad67-447c-bbc1-dcce2eba458e + 2ebb68c0-d2f1-49d1-b4e0-74b8dae0c58f: !Template + answer_choices: 1 ||| 2 + id: 2ebb68c0-d2f1-49d1-b4e0-74b8dae0c58f jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -37,53 +41,35 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A - reference: '' - 013058f2-938f-4370-8894-3e24e9a0937e: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 013058f2-938f-4370-8894-3e24e9a0937e - jinja: ' ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: true_null_prompt - reference: '' - 07f50289-db00-4bd0-a171-ebaa64fd5735: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 07f50289-db00-4bd0-a171-ebaa64fd5735 - jinja: '" ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: null_prompt + name: grammatical_between_1_2 reference: '' - 16efc98a-513f-46b6-b107-bf21926b4162: !Template + 4db2aeb9-59a2-4be6-841e-884a558c6cc6: !Template answer_choices: A ||| B - id: 16efc98a-513f-46b6-b107-bf21926b4162 + id: 4db2aeb9-59a2-4be6-841e-884a558c6cc6 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -97,33 +83,21 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: grammatical_which_one_A_B reference: '' - 23eccce7-ff42-4827-b717-2a0a6b72da0f: !Template + 50f21ae4-7a66-4c3a-81bb-b647f0f2f438: !Template answer_choices: Yes ||| No - id: 23eccce7-ff42-4827-b717-2a0a6b72da0f - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + id: 50f21ae4-7a66-4c3a-81bb-b647f0f2f438 + jinja: '{% set shuffled_order = [0, 1] | random %} - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes - reference: '' - 5e05f264-b2bd-4a11-a3f5-0919b58f88f1: !Template - answer_choices: Yes ||| No - id: 5e05f264-b2bd-4a11-a3f5-0919b58f88f1 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + {% set shuffled_order_options = [0, 1] | random %} - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: @@ -131,24 +105,27 @@ templates: original_task: false name: single_sentence_good_yes_no reference: '' - 6c0bb4da-2905-4b62-a754-841daefcc9e1: !Template - answer_choices: 1 ||| 2 - id: 6c0bb4da-2905-4b62-a754-841daefcc9e1 + 6e06ad25-f429-4088-b92b-2d4b093ed355: !Template + answer_choices: A ||| B + id: 6e06ad25-f429-4088-b92b-2d4b093ed355 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -168,64 +145,33 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 - reference: '' - 7446318d-5013-4e9b-a814-e1fcf3fb5136: !Template - answer_choices: Yes ||| No - id: 7446318d-5013-4e9b-a814-e1fcf3fb5136 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no - reference: '' - a16ed376-1e5c-43fb-8543-cba234ebdec3: !Template - answer_choices: Yes ||| No - id: a16ed376-1e5c-43fb-8543-cba234ebdec3 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes + name: grammatical_between_A_B reference: '' - bc2abaee-e40f-4a02-9221-7e87263d4fb9: !Template + bd29a6c8-dd04-45b5-be84-91936990ef25: !Template answer_choices: A ||| B - id: bc2abaee-e40f-4a02-9221-7e87263d4fb9 + id: bd29a6c8-dd04-45b5-be84-91936990ef25 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -239,65 +185,60 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: better_A_B reference: '' - cbeb4680-e03a-4c13-a8e1-c6527d0cec06: !Template - answer_choices: 1 ||| 2 - id: cbeb4680-e03a-4c13-a8e1-c6527d0cec06 + dad6d138-1f7b-4bd4-96a6-288eaf1a5410: !Template + answer_choices: Yes ||| No + id: dad6d138-1f7b-4bd4-96a6-288eaf1a5410 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. - - {% if shuffled_order == 0 %} + {% set shuffled_order_options = [0, 1] | random %} - 1: {{ sentence_good }} - - 2: {{ sentence_bad }} - - {% else %} - - 1: {{ sentence_bad }} - - 2: {{ sentence_good }} + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} - - {% endif %}' + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + e127adfc-bc9f-4a25-a41c-fcbedd434fdc: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: e127adfc-bc9f-4a25-a41c-fcbedd434fdc + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: null_prompt reference: '' - ea9cd9cb-8c42-4f10-aff0-47a76211cb6f: !Template - answer_choices: A ||| B - id: ea9cd9cb-8c42-4f10-aff0-47a76211cb6f + f74eef44-92da-4edd-86c2-110afd7ba961: !Template + answer_choices: 1 ||| 2 + id: f74eef44-92da-4edd-86c2-110afd7ba961 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -317,5 +258,5 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: grammatical_which_one_1_2 reference: '' diff --git a/promptsource/templates/blimp/existential_there_object_raising/templates.yaml b/promptsource/templates/blimp/existential_there_object_raising/templates.yaml index 35960a05f..7c5ed99da 100644 --- a/promptsource/templates/blimp/existential_there_object_raising/templates.yaml +++ b/promptsource/templates/blimp/existential_there_object_raising/templates.yaml @@ -1,67 +1,35 @@ dataset: blimp subset: existential_there_object_raising templates: - 0678b145-602b-46ce-8229-3e17cec7f729: !Template + 07617b10-d108-4375-9874-85b565cf0db2: !Template answer_choices: Yes ||| No - id: 0678b145-602b-46ce-8229-3e17cec7f729 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no - reference: '' - 069fc06a-bb2a-45cf-aa15-f5effc0b9992: !Template - answer_choices: 1 ||| 2 - id: 069fc06a-bb2a-45cf-aa15-f5effc0b9992 + id: 07617b10-d108-4375-9874-85b565cf0db2 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. - - {% if shuffled_order == 0 %} - - 1: {{ sentence_good }} - - 2: {{ sentence_bad }} - - {% else %} - - 1: {{ sentence_bad }} - - 2: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - {{ answer_choices[0] }} - - {% else %} + {% set shuffled_order_options = [0, 1] | random %} - {{ answer_choices[1] }} + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" - {% endif %}' + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: grammatical_which_one_1_2 + original_task: false + name: single_sentence_good_yes_no reference: '' - 1b50e808-5621-4d8b-bcc5-a6462e1093f7: !Template + 1b94b2ad-1ffe-404b-bf37-ab0c31c975d1: !Template answer_choices: A ||| B - id: 1b50e808-5621-4d8b-bcc5-a6462e1093f7 + id: 1b94b2ad-1ffe-404b-bf37-ab0c31c975d1 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} @@ -93,30 +61,28 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: grammatical_between_A_B reference: '' - 4db5a8c2-b6bf-4f7f-98eb-f0e490ee1f4a: !Template - answer_choices: Yes ||| No - id: 4db5a8c2-b6bf-4f7f-98eb-f0e490ee1f4a - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + 4aacb52f-f2ba-4e54-b8b5-1f6befe8a5d3: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 4aacb52f-f2ba-4e54-b8b5-1f6befe8a5d3 + jinja: ' ||| {{ sentence_good }}' metadata: !TemplateMetadata - choices_in_prompt: true + choices_in_prompt: false metrics: - Accuracy - original_task: false - name: single_sentence_good_yes_no + original_task: true + name: null_prompt reference: '' - 51d1e48e-e478-4754-8ad1-d4a94879a859: !Template + 635017b9-8121-48f5-884b-09cba33bd29d: !Template answer_choices: A ||| B - id: 51d1e48e-e478-4754-8ad1-d4a94879a859 + id: 635017b9-8121-48f5-884b-09cba33bd29d jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} @@ -148,29 +114,15 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B - reference: '' - 5af97eda-fcb2-494a-9ff8-2667b9c45384: !Template - answer_choices: Yes ||| No - id: 5af97eda-fcb2-494a-9ff8-2667b9c45384 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes + name: grammatical_which_one_A_B reference: '' - 66fd650e-857b-4630-acb6-35d9c221f5e0: !Template + 6f0ab739-a964-4da4-aea4-44622fe9c0a3: !Template answer_choices: A ||| B - id: 66fd650e-857b-4630-acb6-35d9c221f5e0 + id: 6f0ab739-a964-4da4-aea4-44622fe9c0a3 jinja: '{% set shuffled_order = [0, 1] | random %} + {% set shuffled_order_options = [0, 1] | random %} + Read the following two sentences. {% if shuffled_order == 0 %} @@ -187,8 +139,8 @@ templates: {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -204,12 +156,35 @@ templates: original_task: true name: better_A_B reference: '' - 6b0ad949-f581-4e96-9e49-ea019e34b56d: !Template + 7c30c862-b19f-4ba4-94a4-62b4ffb395e7: !Template + answer_choices: Yes ||| No + id: 7c30c862-b19f-4ba4-94a4-62b4ffb395e7 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 7e5a3718-a1ad-4767-85b5-f81404a6569c: !Template answer_choices: 1 ||| 2 - id: 6b0ad949-f581-4e96-9e49-ea019e34b56d + id: 7e5a3718-a1ad-4767-85b5-f81404a6569c jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} @@ -243,29 +218,33 @@ templates: original_task: true name: grammatical_between_1_2 reference: '' - 7d778e1e-71ce-4c04-81fb-0910f2df8152: !Template - answer_choices: A ||| B - id: 7d778e1e-71ce-4c04-81fb-0910f2df8152 + c48bf5c5-d91a-450a-9c4e-038da12fff5a: !Template + answer_choices: 1 ||| 2 + id: c48bf5c5-d91a-450a-9c4e-038da12fff5a jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -279,43 +258,5 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A - reference: '' - 8502a14a-57fb-4be7-b6ea-cd954e6c07fc: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 8502a14a-57fb-4be7-b6ea-cd954e6c07fc - jinja: '" ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: null_prompt - reference: '' - ad3b4869-aedb-4660-9c59-fa2d0aee33bf: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: ad3b4869-aedb-4660-9c59-fa2d0aee33bf - jinja: ' ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: true_null_prompt - reference: '' - fd77c0c2-fc51-482a-94d8-b9923c124146: !Template - answer_choices: Yes ||| No - id: fd77c0c2-fc51-482a-94d8-b9923c124146 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes + name: grammatical_which_one_1_2 reference: '' diff --git a/promptsource/templates/blimp/existential_there_quantifiers_1/templates.yaml b/promptsource/templates/blimp/existential_there_quantifiers_1/templates.yaml index 59273a92b..85615e7a4 100644 --- a/promptsource/templates/blimp/existential_there_quantifiers_1/templates.yaml +++ b/promptsource/templates/blimp/existential_there_quantifiers_1/templates.yaml @@ -1,35 +1,27 @@ dataset: blimp subset: existential_there_quantifiers_1 templates: - 00028abe-efa9-4fe7-a933-5c3669a05f69: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 00028abe-efa9-4fe7-a933-5c3669a05f69 - jinja: ' ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: true_null_prompt - reference: '' - 2534a971-4e47-417e-8b5f-190690c614b0: !Template - answer_choices: A ||| B - id: 2534a971-4e47-417e-8b5f-190690c614b0 + 01399a13-0429-4bc3-9fdd-51f448efb5f8: !Template + answer_choices: 1 ||| 2 + id: 01399a13-0429-4bc3-9fdd-51f448efb5f8 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -49,91 +41,48 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: grammatical_which_one_1_2 reference: '' - 2f5a6a22-a6d8-4bb2-9406-86dc5e51a7ed: !Template - answer_choices: A ||| B - id: 2f5a6a22-a6d8-4bb2-9406-86dc5e51a7ed + 2081249e-09b0-43b2-a24d-6ac4ae25eaa7: !Template + answer_choices: Yes ||| No + id: 2081249e-09b0-43b2-a24d-6ac4ae25eaa7 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. - - {% if shuffled_order == 0 %} - - A: {{ sentence_good }} + {% set shuffled_order_options = [0, 1] | random %} - B: {{ sentence_bad }} + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" - {% else %} - - A: {{ sentence_bad }} - - B: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} - - {% endif %}' + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: grammatical_between_A_B + original_task: false + name: single_sentence_good_yes_no reference: '' - 3d8bd633-92b7-411d-9778-2f6882ed2949: !Template - answer_choices: A ||| B - id: 3d8bd633-92b7-411d-9778-2f6882ed2949 - jinja: '{% set shuffled_order = [0, 1] | random %} - - Read the following two sentences. - - {% if shuffled_order == 0 %} - - Sentence A: {{ sentence_good }} - - Sentence B: {{ sentence_bad }} - - {% else %} - - Sentence A: {{ sentence_bad }} - - Sentence B: {{ sentence_good }} - - {% endif %} - - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} - - {% endif %}' + 33871a3f-d1fd-49cd-b485-011df89d7618: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 33871a3f-d1fd-49cd-b485-011df89d7618 + jinja: ' ||| {{ sentence_good }}' metadata: !TemplateMetadata - choices_in_prompt: true + choices_in_prompt: false metrics: - Accuracy original_task: true - name: better_A_B + name: null_prompt reference: '' - 4f6eabc7-c283-453f-b027-78d67a66d863: !Template + 36bed627-38c0-485f-99fc-73f80a254029: !Template answer_choices: 1 ||| 2 - id: 4f6eabc7-c283-453f-b027-78d67a66d863 + id: 36bed627-38c0-485f-99fc-73f80a254029 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} @@ -165,33 +114,21 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: grammatical_between_1_2 reference: '' - 5055533c-854f-4545-a23d-b296d88475a8: !Template + 48a684cf-775d-47f0-ab0d-f8c1de2f4f32: !Template answer_choices: Yes ||| No - id: 5055533c-854f-4545-a23d-b296d88475a8 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + id: 48a684cf-775d-47f0-ab0d-f8c1de2f4f32 + jinja: '{% set shuffled_order = [0, 1] | random %} - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes - reference: '' - 5100e29b-d3b1-4440-8075-1d24c5978a00: !Template - answer_choices: Yes ||| No - id: 5100e29b-d3b1-4440-8075-1d24c5978a00 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: @@ -199,67 +136,67 @@ templates: original_task: false name: single_sentence_bad_yes_no reference: '' - 6d9bd1d3-32ee-4485-8cff-91163197f529: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 6d9bd1d3-32ee-4485-8cff-91163197f529 - jinja: '" ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: null_prompt - reference: '' - 963ef2f1-9270-42b7-9cb5-f83fb23a247b: !Template - answer_choices: Yes ||| No - id: 963ef2f1-9270-42b7-9cb5-f83fb23a247b - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + 84950340-88c5-444f-8e76-e7eab10bfa5a: !Template + answer_choices: A ||| B + id: 84950340-88c5-444f-8e76-e7eab10bfa5a + jinja: '{% set shuffled_order = [0, 1] | random %} - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes - reference: '' - 97add579-6c16-4da4-a267-6f34f9bcf3ab: !Template - answer_choices: Yes ||| No - id: 97add579-6c16-4da4-a267-6f34f9bcf3ab - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: false - name: single_sentence_good_yes_no + original_task: true + name: better_A_B reference: '' - 9a64d9a6-7f4b-4e08-93ed-fd06a2c9e048: !Template - answer_choices: 1 ||| 2 - id: 9a64d9a6-7f4b-4e08-93ed-fd06a2c9e048 + a342b8c6-19f3-47cb-8cc5-cbcb1c2705d7: !Template + answer_choices: A ||| B + id: a342b8c6-19f3-47cb-8cc5-cbcb1c2705d7 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -279,31 +216,35 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: grammatical_which_one_A_B reference: '' - c2bf5161-5e8f-4c19-b50e-cd19f37892bc: !Template + e3e3be3c-df43-45ab-8a2d-604f35e7b9df: !Template answer_choices: A ||| B - id: c2bf5161-5e8f-4c19-b50e-cd19f37892bc + id: e3e3be3c-df43-45ab-8a2d-604f35e7b9df jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -317,5 +258,5 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: grammatical_between_A_B reference: '' diff --git a/promptsource/templates/blimp/existential_there_quantifiers_2/templates.yaml b/promptsource/templates/blimp/existential_there_quantifiers_2/templates.yaml index 35888105e..b62444963 100644 --- a/promptsource/templates/blimp/existential_there_quantifiers_2/templates.yaml +++ b/promptsource/templates/blimp/existential_there_quantifiers_2/templates.yaml @@ -1,12 +1,35 @@ dataset: blimp subset: existential_there_quantifiers_2 templates: - 06081764-6071-4b5b-999b-547ea616d8b8: !Template + 0ce4c191-56f0-44d2-86fe-732ec42a21e0: !Template + answer_choices: Yes ||| No + id: 0ce4c191-56f0-44d2-86fe-732ec42a21e0 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 1d0b5efd-fa90-4f4a-bbec-35d7c0f3c2f1: !Template answer_choices: A ||| B - id: 06081764-6071-4b5b-999b-547ea616d8b8 + id: 1d0b5efd-fa90-4f4a-bbec-35d7c0f3c2f1 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} @@ -38,90 +61,35 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B - reference: '' - 17a25d1e-bae1-4da3-ba3e-10f44cdecc3e: !Template - answer_choices: Yes ||| No - id: 17a25d1e-bae1-4da3-ba3e-10f44cdecc3e - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no - reference: '' - 5bad950a-d854-4711-953c-b0ccb8657d33: !Template - answer_choices: Yes ||| No - id: 5bad950a-d854-4711-953c-b0ccb8657d33 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes - reference: '' - 5c272bae-e167-4887-a6c6-3e57160e69cb: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 5c272bae-e167-4887-a6c6-3e57160e69cb - jinja: '" ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: null_prompt - reference: '' - 78a211a2-bda9-48e7-b08d-193b43f4702d: !Template - answer_choices: Yes ||| No - id: 78a211a2-bda9-48e7-b08d-193b43f4702d - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes + name: grammatical_between_A_B reference: '' - 9366c196-0b41-40a6-8814-2f83cf06bfa2: !Template - answer_choices: A ||| B - id: 9366c196-0b41-40a6-8814-2f83cf06bfa2 + 1e5ddb1b-e417-48a4-a3f1-964f835b4582: !Template + answer_choices: 1 ||| 2 + id: 1e5ddb1b-e417-48a4-a3f1-964f835b4582 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -135,55 +103,32 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: grammatical_which_one_1_2 reference: '' - b9018390-cd5e-4542-826a-0548f9c0a3bf: !Template - answer_choices: A ||| B - id: b9018390-cd5e-4542-826a-0548f9c0a3bf - jinja: '{% set shuffled_order = [0, 1] | random %} - - Read the following two sentences. - - {% if shuffled_order == 0 %} - - Sentence A: {{ sentence_good }} - - Sentence B: {{ sentence_bad }} - - {% else %} - - Sentence A: {{ sentence_bad }} - - Sentence B: {{ sentence_good }} - - {% endif %} - - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} - - {% endif %}' + 33071e39-1731-4e3d-b50b-6d465df9b71d: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 33071e39-1731-4e3d-b50b-6d465df9b71d + jinja: ' ||| {{ sentence_good }}' metadata: !TemplateMetadata - choices_in_prompt: true + choices_in_prompt: false metrics: - Accuracy original_task: true - name: better_A_B + name: null_prompt reference: '' - c5908eee-a950-44e8-a38d-a86df9b8a747: !Template + 33784c12-becc-4b00-955f-a0315357730e: !Template answer_choices: Yes ||| No - id: c5908eee-a950-44e8-a38d-a86df9b8a747 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + id: 33784c12-becc-4b00-955f-a0315357730e + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: @@ -191,35 +136,27 @@ templates: original_task: false name: single_sentence_good_yes_no reference: '' - c5a2336c-4465-470b-88a8-295856d10426: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: c5a2336c-4465-470b-88a8-295856d10426 - jinja: ' ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: true_null_prompt - reference: '' - c6b3931a-868a-4e28-9793-b277e810b366: !Template - answer_choices: A ||| B - id: c6b3931a-868a-4e28-9793-b277e810b366 + 4c1264d8-d80b-47c3-b5a2-3d7f3305cc55: !Template + answer_choices: 1 ||| 2 + id: 4c1264d8-d80b-47c3-b5a2-3d7f3305cc55 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -239,32 +176,33 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: grammatical_between_1_2 reference: '' - de509e9e-dd20-47d6-b502-20522765ac74: !Template - answer_choices: 1 ||| 2 - id: de509e9e-dd20-47d6-b502-20522765ac74 + 6f07a47c-c8c7-4b88-b389-326f5a58c04a: !Template + answer_choices: A ||| B + id: 6f07a47c-c8c7-4b88-b389-326f5a58c04a jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -278,26 +216,29 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: better_A_B reference: '' - eccbb1b2-b932-4a52-bd4d-945ff5c8742c: !Template - answer_choices: 1 ||| 2 - id: eccbb1b2-b932-4a52-bd4d-945ff5c8742c + 829efbf0-b64d-48e1-a9c4-ffff574d3a88: !Template + answer_choices: A ||| B + id: 829efbf0-b64d-48e1-a9c4-ffff574d3a88 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -317,5 +258,5 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: grammatical_which_one_A_B reference: '' diff --git a/promptsource/templates/blimp/existential_there_subject_raising/templates.yaml b/promptsource/templates/blimp/existential_there_subject_raising/templates.yaml index 137b72114..2e7647dc2 100644 --- a/promptsource/templates/blimp/existential_there_subject_raising/templates.yaml +++ b/promptsource/templates/blimp/existential_there_subject_raising/templates.yaml @@ -1,10 +1,10 @@ dataset: blimp subset: existential_there_subject_raising templates: - 386a8e42-071b-450b-a52a-d7e58f298459: !Template + 720eb40e-bd94-4355-a3d7-1a39fcda4d48: !Template answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 386a8e42-071b-450b-a52a-d7e58f298459 - jinja: '" ||| {{ sentence_good }}' + id: 720eb40e-bd94-4355-a3d7-1a39fcda4d48 + jinja: ' ||| {{ sentence_good }}' metadata: !TemplateMetadata choices_in_prompt: false metrics: @@ -12,85 +12,71 @@ templates: original_task: true name: null_prompt reference: '' - 53b18599-30d9-4e99-9d3a-9e6d729a1edd: !Template + 769032ce-d6e8-4456-9996-8935747f0749: !Template answer_choices: Yes ||| No - id: 53b18599-30d9-4e99-9d3a-9e6d729a1edd - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + id: 769032ce-d6e8-4456-9996-8935747f0749 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_good_no_yes + name: single_sentence_good_yes_no reference: '' - 5b565fdc-1f30-48d9-8685-3d0ac35d15cb: !Template - answer_choices: 1 ||| 2 - id: 5b565fdc-1f30-48d9-8685-3d0ac35d15cb + 9b7742fd-42ea-464e-bf01-ac5b87727f22: !Template + answer_choices: Yes ||| No + id: 9b7742fd-42ea-464e-bf01-ac5b87727f22 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. - - {% if shuffled_order == 0 %} - - 1: {{ sentence_good }} - - 2: {{ sentence_bad }} - - {% else %} - - 1: {{ sentence_bad }} - - 2: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - {{ answer_choices[0] }} + {% set shuffled_order_options = [0, 1] | random %} - {% else %} - - {{ answer_choices[1] }} + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" - {% endif %}' + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: grammatical_between_1_2 + original_task: false + name: single_sentence_bad_yes_no reference: '' - 67fdfd3f-a5c8-4358-884a-ca3f5ef6552c: !Template + b1749ae9-3c03-49f2-94f2-db48462aa300: !Template answer_choices: A ||| B - id: 67fdfd3f-a5c8-4358-884a-ca3f5ef6552c + id: b1749ae9-3c03-49f2-94f2-db48462aa300 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -104,63 +90,35 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B - reference: '' - 82ce18ab-4ba9-4be5-81d6-2f0edf50d743: !Template - answer_choices: Yes ||| No - id: 82ce18ab-4ba9-4be5-81d6-2f0edf50d743 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes - reference: '' - adaf26f9-8026-488a-b306-513854c0383d: !Template - answer_choices: Yes ||| No - id: adaf26f9-8026-488a-b306-513854c0383d - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no + name: better_A_B reference: '' - b5259319-925d-47d9-b47f-7dabf6f8d2e9: !Template - answer_choices: A ||| B - id: b5259319-925d-47d9-b47f-7dabf6f8d2e9 + cc088c7d-20d8-493c-aaec-d94e0956bf57: !Template + answer_choices: 1 ||| 2 + id: cc088c7d-20d8-493c-aaec-d94e0956bf57 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -174,31 +132,35 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: grammatical_which_one_1_2 reference: '' - c623eb3d-96f0-4a05-b1d8-f9e02c04510a: !Template + cdb9fa8b-5462-4390-8120-68e9741dcdd1: !Template answer_choices: A ||| B - id: c623eb3d-96f0-4a05-b1d8-f9e02c04510a + id: cdb9fa8b-5462-4390-8120-68e9741dcdd1 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -212,26 +174,29 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: grammatical_which_one_A_B reference: '' - c87e10a4-2113-430a-ab3d-179bff143438: !Template - answer_choices: A ||| B - id: c87e10a4-2113-430a-ab3d-179bff143438 + d85c1524-e607-41b4-8a5f-9487d16489d7: !Template + answer_choices: 1 ||| 2 + id: d85c1524-e607-41b4-8a5f-9487d16489d7 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -251,53 +216,29 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B - reference: '' - d51c69d3-0c0e-409f-b080-093c03dbea5c: !Template - answer_choices: Yes ||| No - id: d51c69d3-0c0e-409f-b080-093c03dbea5c - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no - reference: '' - f0a0beca-ddfd-46d9-a6f5-904b90715b72: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: f0a0beca-ddfd-46d9-a6f5-904b90715b72 - jinja: ' ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: true_null_prompt + name: grammatical_between_1_2 reference: '' - fc64b938-2a21-4f32-8d91-b3fc410cbb94: !Template - answer_choices: 1 ||| 2 - id: fc64b938-2a21-4f32-8d91-b3fc410cbb94 + de2b6985-e35a-4d2c-8b22-0d2edcc080b3: !Template + answer_choices: A ||| B + id: de2b6985-e35a-4d2c-8b22-0d2edcc080b3 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -317,5 +258,5 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: grammatical_between_A_B reference: '' diff --git a/promptsource/templates/blimp/expletive_it_object_raising/templates.yaml b/promptsource/templates/blimp/expletive_it_object_raising/templates.yaml index 0e27ccebc..4f7badbed 100644 --- a/promptsource/templates/blimp/expletive_it_object_raising/templates.yaml +++ b/promptsource/templates/blimp/expletive_it_object_raising/templates.yaml @@ -1,45 +1,33 @@ dataset: blimp subset: expletive_it_object_raising templates: - 30dc01cc-0ffd-4863-abf7-91bdf632a1b2: !Template - answer_choices: Yes ||| No - id: 30dc01cc-0ffd-4863-abf7-91bdf632a1b2 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes - reference: '' - 387a7a2a-2226-4b1e-b420-f73f1fa2f659: !Template - answer_choices: A ||| B - id: 387a7a2a-2226-4b1e-b420-f73f1fa2f659 + 237ae5ee-7bf4-4b16-bd21-b0b144f9f4b6: !Template + answer_choices: 1 ||| 2 + id: 237ae5ee-7bf4-4b16-bd21-b0b144f9f4b6 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -53,17 +41,21 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: grammatical_between_1_2 reference: '' - 5f0000e3-4794-43d6-b141-4df8a99edcea: !Template + 33021819-c858-4356-99e2-9b3ab1b75f0f: !Template answer_choices: Yes ||| No - id: 5f0000e3-4794-43d6-b141-4df8a99edcea - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + id: 33021819-c858-4356-99e2-9b3ab1b75f0f + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: @@ -71,54 +63,19 @@ templates: original_task: false name: single_sentence_good_yes_no reference: '' - 865c9f02-1ce5-44e9-8b65-375cd5a65cec: !Template - answer_choices: 1 ||| 2 - id: 865c9f02-1ce5-44e9-8b65-375cd5a65cec + 54db2e29-0f71-47df-bf44-a117ecb387c9: !Template + answer_choices: Yes ||| No + id: 54db2e29-0f71-47df-bf44-a117ecb387c9 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. - - {% if shuffled_order == 0 %} - - 1: {{ sentence_good }} - - 2: {{ sentence_bad }} - - {% else %} - - 1: {{ sentence_bad }} - - 2: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} + {% set shuffled_order_options = [0, 1] | random %} - {% endif %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: grammatical_which_one_1_2 - reference: '' - 92d10a40-5945-4fbb-9897-e8537776a7af: !Template - answer_choices: Yes ||| No - id: 92d10a40-5945-4fbb-9897-e8537776a7af - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: @@ -126,10 +83,10 @@ templates: original_task: false name: single_sentence_bad_yes_no reference: '' - b19ed231-6114-418d-b09c-834d28a5d982: !Template + 8be709a2-9bc6-4ac8-8171-bd33e944805a: !Template answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: b19ed231-6114-418d-b09c-834d28a5d982 - jinja: '" ||| {{ sentence_good }}' + id: 8be709a2-9bc6-4ac8-8171-bd33e944805a + jinja: ' ||| {{ sentence_good }}' metadata: !TemplateMetadata choices_in_prompt: false metrics: @@ -137,24 +94,27 @@ templates: original_task: true name: null_prompt reference: '' - bd54d076-5b93-43a7-9322-45fdb93206d4: !Template - answer_choices: 1 ||| 2 - id: bd54d076-5b93-43a7-9322-45fdb93206d4 + a4498ce0-f686-42d1-abbb-1880a09d6dc6: !Template + answer_choices: A ||| B + id: a4498ce0-f686-42d1-abbb-1880a09d6dc6 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -174,31 +134,35 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: grammatical_which_one_A_B reference: '' - c30d9409-67bd-4bd2-9366-becbd4b961ed: !Template - answer_choices: A ||| B - id: c30d9409-67bd-4bd2-9366-becbd4b961ed + b05ebbc4-9c71-4a41-9391-a8d82c851671: !Template + answer_choices: 1 ||| 2 + id: b05ebbc4-9c71-4a41-9391-a8d82c851671 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -212,32 +176,33 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: grammatical_which_one_1_2 reference: '' - cbe88ae3-3d99-4e72-b99b-271192c57627: !Template + bcd23eec-fd90-4e8f-bb83-523d67e51b4d: !Template answer_choices: A ||| B - id: cbe88ae3-3d99-4e72-b99b-271192c57627 + id: bcd23eec-fd90-4e8f-bb83-523d67e51b4d jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -251,41 +216,17 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B - reference: '' - d2816a2d-58f2-49b3-a8e0-6a19ec1babfc: !Template - answer_choices: Yes ||| No - id: d2816a2d-58f2-49b3-a8e0-6a19ec1babfc - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes - reference: '' - e18fdda4-907a-4ea6-8865-04e153d880a5: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: e18fdda4-907a-4ea6-8865-04e153d880a5 - jinja: ' ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: true_null_prompt + name: better_A_B reference: '' - ef751a04-c17f-4d8b-8c0e-080d58475c96: !Template + e8e7d70e-4028-4c24-9d29-2cd404252867: !Template answer_choices: A ||| B - id: ef751a04-c17f-4d8b-8c0e-080d58475c96 + id: e8e7d70e-4028-4c24-9d29-2cd404252867 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} @@ -317,5 +258,5 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: grammatical_between_A_B reference: '' diff --git a/promptsource/templates/blimp/inchoative/templates.yaml b/promptsource/templates/blimp/inchoative/templates.yaml index a9cedde75..eca4118ac 100644 --- a/promptsource/templates/blimp/inchoative/templates.yaml +++ b/promptsource/templates/blimp/inchoative/templates.yaml @@ -1,51 +1,27 @@ dataset: blimp subset: inchoative templates: - 01c547f6-f10c-4cf5-9294-68aaf09487f8: !Template - answer_choices: Yes ||| No - id: 01c547f6-f10c-4cf5-9294-68aaf09487f8 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes - reference: '' - 064910ab-4d4c-4033-9c1a-4dfd9d177d70: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 064910ab-4d4c-4033-9c1a-4dfd9d177d70 - jinja: ' ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: true_null_prompt - reference: '' - 25da318f-ebd9-4167-b2eb-ddc5284b4371: !Template - answer_choices: A ||| B - id: 25da318f-ebd9-4167-b2eb-ddc5284b4371 + 17d36910-7d7f-48e8-8841-a67f594279c3: !Template + answer_choices: 1 ||| 2 + id: 17d36910-7d7f-48e8-8841-a67f594279c3 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -65,53 +41,37 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: grammatical_between_1_2 reference: '' - 3e5919ff-4762-48be-9468-55116d6234fe: !Template - answer_choices: 1 ||| 2 - id: 3e5919ff-4762-48be-9468-55116d6234fe + 4d108f13-5ba2-48a5-832d-47e7c67c15f8: !Template + answer_choices: Yes ||| No + id: 4d108f13-5ba2-48a5-832d-47e7c67c15f8 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} - {% if shuffled_order == 0 %} - - 1: {{ sentence_good }} + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" - 2: {{ sentence_bad }} - - {% else %} - - 1: {{ sentence_bad }} - - 2: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} - - {% endif %}' + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: grammatical_which_one_1_2 + original_task: false + name: single_sentence_bad_yes_no reference: '' - 4837edeb-7505-4db8-8465-0d81c89b1b1d: !Template + 54dc5d50-fc3f-48d7-9c9d-269eeae24c71: !Template answer_choices: A ||| B - id: 4837edeb-7505-4db8-8465-0d81c89b1b1d + id: 54dc5d50-fc3f-48d7-9c9d-269eeae24c71 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} @@ -143,31 +103,35 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: grammatical_which_one_A_B reference: '' - 4dc9d016-50f8-4c7e-9329-4adaeeb9abc6: !Template + 65029c1f-8921-42e2-92bd-ac997cf249de: !Template answer_choices: A ||| B - id: 4dc9d016-50f8-4c7e-9329-4adaeeb9abc6 + id: 65029c1f-8921-42e2-92bd-ac997cf249de jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -181,41 +145,17 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B - reference: '' - 59b356d9-53ca-491c-820f-e7b2b7eb1274: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 59b356d9-53ca-491c-820f-e7b2b7eb1274 - jinja: '" ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: null_prompt - reference: '' - 6a549a6e-f2da-48f7-b565-82fe320c7067: !Template - answer_choices: Yes ||| No - id: 6a549a6e-f2da-48f7-b565-82fe320c7067 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes + name: grammatical_between_A_B reference: '' - 6f77d0b0-4def-44cb-8989-94cf63ec1302: !Template + 76b778db-8193-44bf-b89d-357bce08373a: !Template answer_choices: 1 ||| 2 - id: 6f77d0b0-4def-44cb-8989-94cf63ec1302 + id: 76b778db-8193-44bf-b89d-357bce08373a jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} @@ -247,45 +187,15 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 - reference: '' - 7198da05-87f2-4fb6-954e-317f6951c56d: !Template - answer_choices: Yes ||| No - id: 7198da05-87f2-4fb6-954e-317f6951c56d - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no - reference: '' - 724338e9-04f1-494c-b056-498855abcfee: !Template - answer_choices: Yes ||| No - id: 724338e9-04f1-494c-b056-498855abcfee - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no + name: grammatical_which_one_1_2 reference: '' - c28c1ceb-77b8-43b4-ba2f-6f078f0a3f30: !Template + 7db4bd8d-42c0-44da-abb5-7228b128f0d3: !Template answer_choices: A ||| B - id: c28c1ceb-77b8-43b4-ba2f-6f078f0a3f30 + id: 7db4bd8d-42c0-44da-abb5-7228b128f0d3 jinja: '{% set shuffled_order = [0, 1] | random %} + {% set shuffled_order_options = [0, 1] | random %} + Read the following two sentences. {% if shuffled_order == 0 %} @@ -302,8 +212,8 @@ templates: {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -317,5 +227,36 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: better_A_B + reference: '' + c5de2c20-822b-4676-8645-8f7140c224c8: !Template + answer_choices: Yes ||| No + id: c5de2c20-822b-4676-8645-8f7140c224c8 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + e7765e43-44c5-487c-a968-4cc931598e34: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: e7765e43-44c5-487c-a968-4cc931598e34 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt reference: '' diff --git a/promptsource/templates/blimp/intransitive/templates.yaml b/promptsource/templates/blimp/intransitive/templates.yaml index f06f8a0a3..fb8fc6748 100644 --- a/promptsource/templates/blimp/intransitive/templates.yaml +++ b/promptsource/templates/blimp/intransitive/templates.yaml @@ -1,61 +1,35 @@ dataset: blimp subset: intransitive templates: - 3c912318-b915-4cd4-a921-294092393382: !Template - answer_choices: A ||| B - id: 3c912318-b915-4cd4-a921-294092393382 + 09eccac1-7d0d-4cbf-839f-3815d6b35c3a: !Template + answer_choices: Yes ||| No + id: 09eccac1-7d0d-4cbf-839f-3815d6b35c3a jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. - - {% if shuffled_order == 0 %} - - Sentence A: {{ sentence_good }} + {% set shuffled_order_options = [0, 1] | random %} - Sentence B: {{ sentence_bad }} - - {% else %} - - Sentence A: {{ sentence_bad }} - - Sentence B: {{ sentence_good }} - - {% endif %} - - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} - - {{ answer_choices[0] }} + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" - {% else %} - - {{ answer_choices[1] }} - - {% endif %}' + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: better_A_B - reference: '' - 4b3be9c6-383a-4be9-8caf-df394a404632: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 4b3be9c6-383a-4be9-8caf-df394a404632 - jinja: '" ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: null_prompt + original_task: false + name: single_sentence_good_yes_no reference: '' - 4d8e4dd3-fe6f-4450-ab3e-ca775388ff6c: !Template + 29b6507a-dab9-4e6e-b644-f6896d67091a: !Template answer_choices: 1 ||| 2 - id: 4d8e4dd3-fe6f-4450-ab3e-ca775388ff6c + id: 29b6507a-dab9-4e6e-b644-f6896d67091a jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} @@ -87,32 +61,33 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: grammatical_which_one_1_2 reference: '' - 6ebec8d2-e5c0-4f76-bb46-3665befc5dc4: !Template - answer_choices: 1 ||| 2 - id: 6ebec8d2-e5c0-4f76-bb46-3665befc5dc4 + 4f61d91b-ef03-4700-80be-74c719446815: !Template + answer_choices: A ||| B + id: 4f61d91b-ef03-4700-80be-74c719446815 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -126,14 +101,17 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: better_A_B reference: '' - 87a67a6f-f043-4d6c-9c51-7ac61df3b55a: !Template + 7d082f4d-f8cb-407b-92bc-18b0ea5bb8d1: !Template answer_choices: A ||| B - id: 87a67a6f-f043-4d6c-9c51-7ac61df3b55a + id: 7d082f4d-f8cb-407b-92bc-18b0ea5bb8d1 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} @@ -167,24 +145,58 @@ templates: original_task: true name: grammatical_which_one_A_B reference: '' - a67f5769-edcd-45e7-a930-7ea6849c2e2e: !Template - answer_choices: A ||| B - id: a67f5769-edcd-45e7-a930-7ea6849c2e2e + 82e180ac-5afe-49cc-afb2-8b4dd4c609c1: !Template + answer_choices: Yes ||| No + id: 82e180ac-5afe-49cc-afb2-8b4dd4c609c1 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 947578c4-143a-4062-a476-e9a39f7f8020: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 947578c4-143a-4062-a476-e9a39f7f8020 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + a5fa1d8e-cd6e-448a-9b97-a3331594a9aa: !Template + answer_choices: 1 ||| 2 + id: a5fa1d8e-cd6e-448a-9b97-a3331594a9aa + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -204,74 +216,35 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B - reference: '' - b4069486-a61e-41b4-b4f6-1524ff686a66: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: b4069486-a61e-41b4-b4f6-1524ff686a66 - jinja: ' ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: true_null_prompt - reference: '' - c7b062bf-7ed7-40c4-84ad-1ec9e3ef51a7: !Template - answer_choices: Yes ||| No - id: c7b062bf-7ed7-40c4-84ad-1ec9e3ef51a7 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no - reference: '' - d2f641f2-1a2a-474c-85c5-1d8d49f4a103: !Template - answer_choices: Yes ||| No - id: d2f641f2-1a2a-474c-85c5-1d8d49f4a103 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes + name: grammatical_between_1_2 reference: '' - dbb33462-77f8-4e6c-947d-0cabd5f9920f: !Template + bc749983-4028-4466-947b-bc021f616fa0: !Template answer_choices: A ||| B - id: dbb33462-77f8-4e6c-947d-0cabd5f9920f + id: bc749983-4028-4466-947b-bc021f616fa0 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -285,37 +258,5 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A - reference: '' - e116f131-9021-4f6d-8176-38b58f19058c: !Template - answer_choices: Yes ||| No - id: e116f131-9021-4f6d-8176-38b58f19058c - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes - reference: '' - fba1ef34-6f4e-4f19-9bd4-7b01181e2e4b: !Template - answer_choices: Yes ||| No - id: fba1ef34-6f4e-4f19-9bd4-7b01181e2e4b - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no + name: grammatical_between_A_B reference: '' diff --git a/promptsource/templates/blimp/irregular_past_participle_adjectives/templates.yaml b/promptsource/templates/blimp/irregular_past_participle_adjectives/templates.yaml index 2261dd41c..f51ab26e6 100644 --- a/promptsource/templates/blimp/irregular_past_participle_adjectives/templates.yaml +++ b/promptsource/templates/blimp/irregular_past_participle_adjectives/templates.yaml @@ -1,67 +1,15 @@ dataset: blimp subset: irregular_past_participle_adjectives templates: - 16a8523d-2f5c-4d89-aedf-6f2a79a9bfec: !Template - answer_choices: A ||| B - id: 16a8523d-2f5c-4d89-aedf-6f2a79a9bfec - jinja: '{% set shuffled_order = [0, 1] | random %} - - Which one of the following sentences is grammatical? Please answer A or B. - - {% if shuffled_order == 0 %} - - A: {{ sentence_good }} - - B: {{ sentence_bad }} - - {% else %} - - A: {{ sentence_bad }} - - B: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} - - {% endif %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: grammatical_which_one_A_B - reference: '' - 1d494a1b-ee23-4ffb-bcfb-4e1a160bf245: !Template - answer_choices: Yes ||| No - id: 1d494a1b-ee23-4ffb-bcfb-4e1a160bf245 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes - reference: '' - 2b880aa1-25b4-4694-8ca9-efd89b51e86a: !Template + 01c8bd41-9a1f-4030-a4ab-4b21ac314a21: !Template answer_choices: 1 ||| 2 - id: 2b880aa1-25b4-4694-8ca9-efd89b51e86a + id: 01c8bd41-9a1f-4030-a4ab-4b21ac314a21 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} @@ -93,44 +41,12 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 - reference: '' - 60e8eab4-8c21-4ef4-9141-196e2c89be04: !Template - answer_choices: Yes ||| No - id: 60e8eab4-8c21-4ef4-9141-196e2c89be04 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no - reference: '' - 68594837-019a-4625-8cc3-bc00e853e757: !Template - answer_choices: Yes ||| No - id: 68594837-019a-4625-8cc3-bc00e853e757 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes + name: grammatical_which_one_1_2 reference: '' - 7465ee8b-594f-43d7-afc1-a029b99cec06: !Template + 20ceb8f1-af0f-44eb-b06c-7a64715d4165: !Template answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 7465ee8b-594f-43d7-afc1-a029b99cec06 - jinja: '" ||| {{ sentence_good }}' + id: 20ceb8f1-af0f-44eb-b06c-7a64715d4165 + jinja: ' ||| {{ sentence_good }}' metadata: !TemplateMetadata choices_in_prompt: false metrics: @@ -138,22 +54,33 @@ templates: original_task: true name: null_prompt reference: '' - 7c16eb5f-05b6-4e73-9fb6-16f91ad1fbb1: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 7c16eb5f-05b6-4e73-9fb6-16f91ad1fbb1 - jinja: ' ||| {{ sentence_good }}' + 3b5e3a7e-a847-4a20-af6e-ae94776f15a9: !Template + answer_choices: Yes ||| No + id: 3b5e3a7e-a847-4a20-af6e-ae94776f15a9 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata - choices_in_prompt: false + choices_in_prompt: true metrics: - Accuracy - original_task: true - name: true_null_prompt + original_task: false + name: single_sentence_bad_yes_no reference: '' - 87516bcf-34f6-4fde-8142-e9340a54e32c: !Template + 4a4440cb-a0ba-49f8-89b8-44f29a0abaac: !Template answer_choices: A ||| B - id: 87516bcf-34f6-4fde-8142-e9340a54e32c + id: 4a4440cb-a0ba-49f8-89b8-44f29a0abaac jinja: '{% set shuffled_order = [0, 1] | random %} + {% set shuffled_order_options = [0, 1] | random %} + Read the following two sentences. {% if shuffled_order == 0 %} @@ -170,8 +97,8 @@ templates: {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -187,29 +114,53 @@ templates: original_task: true name: better_A_B reference: '' - ca78195f-5b74-4046-9c13-3c1ce1b0c663: !Template + 60ecd607-6fa1-4163-b6e4-15ae731edad5: !Template + answer_choices: Yes ||| No + id: 60ecd607-6fa1-4163-b6e4-15ae731edad5 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 7c796d3f-205d-444c-a701-57d52ff9fc4a: !Template answer_choices: A ||| B - id: ca78195f-5b74-4046-9c13-3c1ce1b0c663 + id: 7c796d3f-205d-444c-a701-57d52ff9fc4a jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -223,42 +174,29 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A - reference: '' - e4c06619-699c-4a12-a3d7-78cc606c199b: !Template - answer_choices: Yes ||| No - id: e4c06619-699c-4a12-a3d7-78cc606c199b - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no + name: grammatical_which_one_A_B reference: '' - e5daf178-7934-405c-9c83-35aeedc95fed: !Template - answer_choices: 1 ||| 2 - id: e5daf178-7934-405c-9c83-35aeedc95fed + b0716af9-24a4-4d00-8be6-7c61de12bdf2: !Template + answer_choices: A ||| B + id: b0716af9-24a4-4d00-8be6-7c61de12bdf2 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -278,26 +216,29 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: grammatical_between_A_B reference: '' - e917d2a5-66ae-4faf-be0d-1f9ad8201d7c: !Template - answer_choices: A ||| B - id: e917d2a5-66ae-4faf-be0d-1f9ad8201d7c + d9490bb4-8fb6-458d-94fc-c8918d851240: !Template + answer_choices: 1 ||| 2 + id: d9490bb4-8fb6-458d-94fc-c8918d851240 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -317,5 +258,5 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: grammatical_between_1_2 reference: '' diff --git a/promptsource/templates/blimp/irregular_past_participle_verbs/templates.yaml b/promptsource/templates/blimp/irregular_past_participle_verbs/templates.yaml index 9b2df8d9c..54e0950ed 100644 --- a/promptsource/templates/blimp/irregular_past_participle_verbs/templates.yaml +++ b/promptsource/templates/blimp/irregular_past_participle_verbs/templates.yaml @@ -1,99 +1,15 @@ dataset: blimp subset: irregular_past_participle_verbs templates: - 01795e7b-f8b1-45a0-8ae1-f1c0e3b21adb: !Template - answer_choices: Yes ||| No - id: 01795e7b-f8b1-45a0-8ae1-f1c0e3b21adb - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes - reference: '' - 048a9ad5-6cd1-4927-a9c2-3682d1c3dfe6: !Template - answer_choices: 1 ||| 2 - id: 048a9ad5-6cd1-4927-a9c2-3682d1c3dfe6 - jinja: '{% set shuffled_order = [0, 1] | random %} - - Between the two sentences, which one is grammatical? Please answer 1 or 2. - - {% if shuffled_order == 0 %} - - 1: {{ sentence_good }} - - 2: {{ sentence_bad }} - - {% else %} - - 1: {{ sentence_bad }} - - 2: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} - - {% endif %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: grammatical_between_1_2 - reference: '' - 3a6f271c-7d0c-4586-b475-54c32ce98e5b: !Template - answer_choices: Yes ||| No - id: 3a6f271c-7d0c-4586-b475-54c32ce98e5b - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no - reference: '' - 79827cfd-939e-4bed-938a-c511e4e74446: !Template - answer_choices: Yes ||| No - id: 79827cfd-939e-4bed-938a-c511e4e74446 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes - reference: '' - 7a1e655c-f284-4966-a5f3-df64f9ec6ba0: !Template + 033b7e11-463c-4a4f-a7e4-3010e8042b63: !Template answer_choices: A ||| B - id: 7a1e655c-f284-4966-a5f3-df64f9ec6ba0 + id: 033b7e11-463c-4a4f-a7e4-3010e8042b63 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} @@ -125,51 +41,15 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B - reference: '' - 89318530-c50e-43ba-a7fc-528a8b23f8c1: !Template - answer_choices: Yes ||| No - id: 89318530-c50e-43ba-a7fc-528a8b23f8c1 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no - reference: '' - 92fc33a6-dec5-44a2-b12a-1550ebaaac17: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 92fc33a6-dec5-44a2-b12a-1550ebaaac17 - jinja: '" ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: null_prompt - reference: '' - 9aecc761-2e55-4553-afae-ead346c4e6b4: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 9aecc761-2e55-4553-afae-ead346c4e6b4 - jinja: ' ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: true_null_prompt + name: grammatical_which_one_A_B reference: '' - c9ebec28-a6fd-4743-b320-4bafffc2de86: !Template + 094ad547-7da1-4405-b8c4-232b68406bb6: !Template answer_choices: A ||| B - id: c9ebec28-a6fd-4743-b320-4bafffc2de86 + id: 094ad547-7da1-4405-b8c4-232b68406bb6 jinja: '{% set shuffled_order = [0, 1] | random %} + {% set shuffled_order_options = [0, 1] | random %} + Read the following two sentences. {% if shuffled_order == 0 %} @@ -186,8 +66,8 @@ templates: {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -201,26 +81,29 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: better_A_B reference: '' - cb56e5d4-2628-4c31-9b73-fb7cdb078cc5: !Template - answer_choices: A ||| B - id: cb56e5d4-2628-4c31-9b73-fb7cdb078cc5 + 1ca9115d-82a3-4e9a-80ce-4b095b391f6c: !Template + answer_choices: 1 ||| 2 + id: 1ca9115d-82a3-4e9a-80ce-4b095b391f6c jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -240,31 +123,55 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: grammatical_between_1_2 reference: '' - d6ce7716-6632-45dc-b9ae-3395d6f8e728: !Template - answer_choices: A ||| B - id: d6ce7716-6632-45dc-b9ae-3395d6f8e728 + 7455d78b-7d69-4c26-af9f-280078cab5c8: !Template + answer_choices: Yes ||| No + id: 7455d78b-7d69-4c26-af9f-280078cab5c8 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 8d7922a1-5922-412a-ba40-7166ca83f863: !Template + answer_choices: 1 ||| 2 + id: 8d7922a1-5922-412a-ba40-7166ca83f863 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -278,26 +185,29 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: grammatical_which_one_1_2 reference: '' - ff56058b-27f8-4c79-b1d5-89c56b9c1cf5: !Template - answer_choices: 1 ||| 2 - id: ff56058b-27f8-4c79-b1d5-89c56b9c1cf5 + a2e0f26c-aea2-4ebf-88bd-b40f9a8015ba: !Template + answer_choices: A ||| B + id: a2e0f26c-aea2-4ebf-88bd-b40f9a8015ba jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -317,5 +227,36 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: grammatical_between_A_B + reference: '' + d309555f-4f32-4288-bfcd-eebcd1dc5507: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: d309555f-4f32-4288-bfcd-eebcd1dc5507 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + e98ee2cf-a98c-460d-bdd2-c967fc750d0f: !Template + answer_choices: Yes ||| No + id: e98ee2cf-a98c-460d-bdd2-c967fc750d0f + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no reference: '' diff --git a/promptsource/templates/blimp/irregular_plural_subject_verb_agreement_1/templates.yaml b/promptsource/templates/blimp/irregular_plural_subject_verb_agreement_1/templates.yaml index d4916b2e5..5efd68937 100644 --- a/promptsource/templates/blimp/irregular_plural_subject_verb_agreement_1/templates.yaml +++ b/promptsource/templates/blimp/irregular_plural_subject_verb_agreement_1/templates.yaml @@ -1,15 +1,19 @@ dataset: blimp subset: irregular_plural_subject_verb_agreement_1 templates: - 2a951848-458f-4dad-ab47-f8eac9a74d0a: !Template + 18e38f65-c6c5-4496-a5da-38b8f10403f6: !Template answer_choices: Yes ||| No - id: 2a951848-458f-4dad-ab47-f8eac9a74d0a - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + id: 18e38f65-c6c5-4496-a5da-38b8f10403f6 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: @@ -17,23 +21,15 @@ templates: original_task: false name: single_sentence_good_yes_no reference: '' - 3a753455-5153-4bb3-8249-66c359466110: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 3a753455-5153-4bb3-8249-66c359466110 - jinja: '" ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: null_prompt - reference: '' - 42da8e53-8363-48d0-b793-6922ad672ba0: !Template + 1ac85c8f-7341-41e8-b9ff-4639a6be27c2: !Template answer_choices: A ||| B - id: 42da8e53-8363-48d0-b793-6922ad672ba0 + id: 1ac85c8f-7341-41e8-b9ff-4639a6be27c2 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} @@ -65,43 +61,33 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B - reference: '' - 7596b6e3-9265-4cd3-bb43-49555416a253: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 7596b6e3-9265-4cd3-bb43-49555416a253 - jinja: ' ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: true_null_prompt + name: grammatical_which_one_A_B reference: '' - 77575034-c0b0-42e4-8423-8e8543085834: !Template + 2530993f-5703-40af-bfba-0dcc3ee8a8f7: !Template answer_choices: A ||| B - id: 77575034-c0b0-42e4-8423-8e8543085834 + id: 2530993f-5703-40af-bfba-0dcc3ee8a8f7 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -115,30 +101,17 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B - reference: '' - 7c15530c-1869-428a-975b-46c052055b62: !Template - answer_choices: Yes ||| No - id: 7c15530c-1869-428a-975b-46c052055b62 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes + name: better_A_B reference: '' - 8061c884-5f43-4549-a3e4-c1acb736931b: !Template + 3f258e68-f24f-4679-a1b0-c67d1af2d346: !Template answer_choices: 1 ||| 2 - id: 8061c884-5f43-4549-a3e4-c1acb736931b + id: 3f258e68-f24f-4679-a1b0-c67d1af2d346 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} @@ -170,46 +143,17 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 - reference: '' - 80fc5d64-dbac-46af-b42a-af73349215f5: !Template - answer_choices: Yes ||| No - id: 80fc5d64-dbac-46af-b42a-af73349215f5 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no - reference: '' - 8c24a79f-adc4-4ac7-9ff4-b73f0b08dcde: !Template - answer_choices: Yes ||| No - id: 8c24a79f-adc4-4ac7-9ff4-b73f0b08dcde - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes + name: grammatical_which_one_1_2 reference: '' - 8d93abf0-ef5f-4260-b3cb-eccaabcfb3b5: !Template + 78ef0745-70c5-4aeb-a29f-231a15e88842: !Template answer_choices: 1 ||| 2 - id: 8d93abf0-ef5f-4260-b3cb-eccaabcfb3b5 + id: 78ef0745-70c5-4aeb-a29f-231a15e88842 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} @@ -241,69 +185,66 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: grammatical_between_1_2 reference: '' - a6d2c08b-d883-4d8c-b1de-c47412ffc5a7: !Template - answer_choices: A ||| B - id: a6d2c08b-d883-4d8c-b1de-c47412ffc5a7 + d5e3d605-0b6d-4a2a-a75e-12fede0bc849: !Template + answer_choices: Yes ||| No + id: d5e3d605-0b6d-4a2a-a75e-12fede0bc849 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. - - {% if shuffled_order == 0 %} - - Sentence A: {{ sentence_good }} - - Sentence B: {{ sentence_bad }} - - {% else %} - - Sentence A: {{ sentence_bad }} - - Sentence B: {{ sentence_good }} - - {% endif %} - - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} - - {{ answer_choices[0] }} - - {% else %} + {% set shuffled_order_options = [0, 1] | random %} - {{ answer_choices[1] }} + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" - {% endif %}' + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + dbfdec5c-9a34-4121-81d9-61978296d0fb: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: dbfdec5c-9a34-4121-81d9-61978296d0fb + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy original_task: true - name: better_B_A + name: null_prompt reference: '' - d5548e5c-998b-4f60-96c4-be23b9ace2e0: !Template + f3c61507-8558-4b9d-9c5b-0476563b553a: !Template answer_choices: A ||| B - id: d5548e5c-998b-4f60-96c4-be23b9ace2e0 + id: f3c61507-8558-4b9d-9c5b-0476563b553a jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -317,5 +258,5 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: grammatical_between_A_B reference: '' diff --git a/promptsource/templates/blimp/irregular_plural_subject_verb_agreement_2/templates.yaml b/promptsource/templates/blimp/irregular_plural_subject_verb_agreement_2/templates.yaml index bfb288562..469d8aeb7 100644 --- a/promptsource/templates/blimp/irregular_plural_subject_verb_agreement_2/templates.yaml +++ b/promptsource/templates/blimp/irregular_plural_subject_verb_agreement_2/templates.yaml @@ -1,93 +1,35 @@ dataset: blimp subset: irregular_plural_subject_verb_agreement_2 templates: - 4bc70b39-ec97-40a0-aa08-e8323cff9367: !Template + 6a203fde-56e9-42bb-bae9-639748ff420a: !Template answer_choices: Yes ||| No - id: 4bc70b39-ec97-40a0-aa08-e8323cff9367 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no - reference: '' - 54a29275-7b43-4af6-8bae-dcb2d51d0e6c: !Template - answer_choices: Yes ||| No - id: 54a29275-7b43-4af6-8bae-dcb2d51d0e6c - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes - reference: '' - 5bfd948d-2a9e-447b-954f-ee7c04daa5f8: !Template - answer_choices: A ||| B - id: 5bfd948d-2a9e-447b-954f-ee7c04daa5f8 + id: 6a203fde-56e9-42bb-bae9-639748ff420a jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + {% set shuffled_order_options = [0, 1] | random %} - {% if shuffled_order == 0 %} - - Sentence A: {{ sentence_good }} - - Sentence B: {{ sentence_bad }} + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" - {% else %} - - Sentence A: {{ sentence_bad }} - - Sentence B: {{ sentence_good }} - - {% endif %} - - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} - - {% endif %}' + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: better_A_B - reference: '' - 632d0460-942a-4d0f-970f-2a6e22fda4f6: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 632d0460-942a-4d0f-970f-2a6e22fda4f6 - jinja: '" ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: null_prompt + original_task: false + name: single_sentence_bad_yes_no reference: '' - 8ad81c4e-36a5-481f-be12-7283effb4e05: !Template + 95c16bb7-58e4-499f-a4cb-aec35dbb5505: !Template answer_choices: 1 ||| 2 - id: 8ad81c4e-36a5-481f-be12-7283effb4e05 + id: 95c16bb7-58e4-499f-a4cb-aec35dbb5505 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} @@ -119,13 +61,15 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: grammatical_which_one_1_2 reference: '' - 8fbb9f54-2881-4e5c-b83f-8827a288d5db: !Template + a0ef739a-6f04-4529-ae50-cc146355e526: !Template answer_choices: A ||| B - id: 8fbb9f54-2881-4e5c-b83f-8827a288d5db + id: a0ef739a-6f04-4529-ae50-cc146355e526 jinja: '{% set shuffled_order = [0, 1] | random %} + {% set shuffled_order_options = [0, 1] | random %} + Read the following two sentences. {% if shuffled_order == 0 %} @@ -142,8 +86,8 @@ templates: {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -157,14 +101,17 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: better_A_B reference: '' - a77827bc-d9e1-4003-aba0-e3c7a3b86f1c: !Template + af407614-1c7b-424f-acfe-37ab790703f3: !Template answer_choices: 1 ||| 2 - id: a77827bc-d9e1-4003-aba0-e3c7a3b86f1c + id: af407614-1c7b-424f-acfe-37ab790703f3 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} @@ -196,14 +143,28 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: grammatical_between_1_2 + reference: '' + ca6d5112-5d25-4611-9886-79618bba1afa: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: ca6d5112-5d25-4611-9886-79618bba1afa + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt reference: '' - b9ab8131-0d86-448e-8a84-dc102b61de33: !Template + d7b194b9-39c6-4ae8-a85f-d8bdb961bc93: !Template answer_choices: A ||| B - id: b9ab8131-0d86-448e-8a84-dc102b61de33 + id: d7b194b9-39c6-4ae8-a85f-d8bdb961bc93 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} @@ -237,12 +198,15 @@ templates: original_task: true name: grammatical_between_A_B reference: '' - d4d5d236-eb1a-449e-a693-3443ca550070: !Template + eea5f683-71fc-4660-a844-ab5ea175c740: !Template answer_choices: A ||| B - id: d4d5d236-eb1a-449e-a693-3443ca550070 + id: eea5f683-71fc-4660-a844-ab5ea175c740 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} @@ -276,46 +240,23 @@ templates: original_task: true name: grammatical_which_one_A_B reference: '' - dc551883-e804-49c7-bcc1-d9debef56b5d: !Template + f8ac205b-8848-4c70-b13b-7148f3c09294: !Template answer_choices: Yes ||| No - id: dc551883-e804-49c7-bcc1-d9debef56b5d - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + id: f8ac205b-8848-4c70-b13b-7148f3c09294 + jinja: '{% set shuffled_order = [0, 1] | random %} - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no - reference: '' - f2c4adf3-85af-4e31-9656-313fdc77287e: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: f2c4adf3-85af-4e31-9656-313fdc77287e - jinja: ' ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: true_null_prompt - reference: '' - f9647cd9-b31f-4d72-a741-96b8e8374458: !Template - answer_choices: Yes ||| No - id: f9647cd9-b31f-4d72-a741-96b8e8374458 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + {% set shuffled_order_options = [0, 1] | random %} - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_good_no_yes + name: single_sentence_good_yes_no reference: '' diff --git a/promptsource/templates/blimp/left_branch_island_echo_question/templates.yaml b/promptsource/templates/blimp/left_branch_island_echo_question/templates.yaml index eade59b46..f10cc606e 100644 --- a/promptsource/templates/blimp/left_branch_island_echo_question/templates.yaml +++ b/promptsource/templates/blimp/left_branch_island_echo_question/templates.yaml @@ -1,93 +1,66 @@ dataset: blimp subset: left_branch_island_echo_question templates: - 3a8adcf5-1de5-44df-86dc-5d4180aed887: !Template - answer_choices: A ||| B - id: 3a8adcf5-1de5-44df-86dc-5d4180aed887 - jinja: '{% set shuffled_order = [0, 1] | random %} - - Read the following two sentences. - - {% if shuffled_order == 0 %} - - Sentence A: {{ sentence_good }} - - Sentence B: {{ sentence_bad }} - - {% else %} - - Sentence A: {{ sentence_bad }} - - Sentence B: {{ sentence_good }} - - {% endif %} - - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} - - {% endif %}' + 309c0c8a-f7b0-475a-8fd0-65fba3ece1bb: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 309c0c8a-f7b0-475a-8fd0-65fba3ece1bb + jinja: ' ||| {{ sentence_good }}' metadata: !TemplateMetadata - choices_in_prompt: true + choices_in_prompt: false metrics: - Accuracy original_task: true - name: better_B_A + name: null_prompt reference: '' - 455f42d8-caa8-4890-9545-607d3ef32e90: !Template + 7da51f1d-f891-48e6-b271-b9602d7eb08b: !Template answer_choices: Yes ||| No - id: 455f42d8-caa8-4890-9545-607d3ef32e90 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + id: 7da51f1d-f891-48e6-b271-b9602d7eb08b + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_good_no_yes + name: single_sentence_good_yes_no reference: '' - 7b225c23-9765-44e5-b051-be845dddeeed: !Template + c0402910-d21b-4dc2-ab00-6823d349c3a8: !Template answer_choices: Yes ||| No - id: 7b225c23-9765-44e5-b051-be845dddeeed - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + id: c0402910-d21b-4dc2-ab00-6823d349c3a8 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_good_yes_no - reference: '' - 95b23722-4db7-4283-8937-1f2ba5e67f25: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 95b23722-4db7-4283-8937-1f2ba5e67f25 - jinja: '" ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: null_prompt + name: single_sentence_bad_yes_no reference: '' - 9eee0140-1bb2-40cc-a363-1f24dd0de19c: !Template + e10efdbb-6452-4446-befd-42bff2da8d51: !Template answer_choices: 1 ||| 2 - id: 9eee0140-1bb2-40cc-a363-1f24dd0de19c + id: e10efdbb-6452-4446-befd-42bff2da8d51 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} @@ -119,37 +92,29 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 - reference: '' - a8a33207-5db2-4f02-bce1-347d6da4c77e: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: a8a33207-5db2-4f02-bce1-347d6da4c77e - jinja: ' ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: true_null_prompt + name: grammatical_between_1_2 reference: '' - aee17b77-9972-42a5-8e2b-d9893bfcdfe8: !Template - answer_choices: 1 ||| 2 - id: aee17b77-9972-42a5-8e2b-d9893bfcdfe8 + e7a31ef5-2205-4deb-a5d7-6d872b85f55d: !Template + answer_choices: A ||| B + id: e7a31ef5-2205-4deb-a5d7-6d872b85f55d jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -169,48 +134,33 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 - reference: '' - ba0af8f4-f5c5-4859-bab5-e43463e47c85: !Template - answer_choices: Yes ||| No - id: ba0af8f4-f5c5-4859-bab5-e43463e47c85 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no + name: grammatical_between_A_B reference: '' - db6a7631-9bfb-4c51-ad55-39bea946c4a6: !Template + f052b636-8fdc-45e4-aaad-10147b6e28be: !Template answer_choices: A ||| B - id: db6a7631-9bfb-4c51-ad55-39bea946c4a6 + id: f052b636-8fdc-45e4-aaad-10147b6e28be jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -224,31 +174,35 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: better_A_B reference: '' - ed86f3fd-fcec-4f33-8298-d0eeeb82051c: !Template - answer_choices: A ||| B - id: ed86f3fd-fcec-4f33-8298-d0eeeb82051c + f61cf914-d4d2-47d5-bc18-4196932de519: !Template + answer_choices: 1 ||| 2 + id: f61cf914-d4d2-47d5-bc18-4196932de519 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -262,30 +216,17 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B - reference: '' - fa45e88d-fa9b-4165-803c-03aa61295cce: !Template - answer_choices: Yes ||| No - id: fa45e88d-fa9b-4165-803c-03aa61295cce - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes + name: grammatical_which_one_1_2 reference: '' - fcb7c59f-0e54-40c3-a444-ddacf0d67c1f: !Template + fafa04c4-3b6a-49fb-9cb2-7a1e6f2285be: !Template answer_choices: A ||| B - id: fcb7c59f-0e54-40c3-a444-ddacf0d67c1f + id: fafa04c4-3b6a-49fb-9cb2-7a1e6f2285be jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} diff --git a/promptsource/templates/blimp/left_branch_island_simple_question/templates.yaml b/promptsource/templates/blimp/left_branch_island_simple_question/templates.yaml index 7964991ec..4fa8d5b05 100644 --- a/promptsource/templates/blimp/left_branch_island_simple_question/templates.yaml +++ b/promptsource/templates/blimp/left_branch_island_simple_question/templates.yaml @@ -1,62 +1,27 @@ dataset: blimp subset: left_branch_island_simple_question templates: - 08143a2e-5071-49c6-800b-dc41b3053d6c: !Template + 12dc4d87-ea95-4f68-ac86-b6b17d5448b5: !Template answer_choices: A ||| B - id: 08143a2e-5071-49c6-800b-dc41b3053d6c + id: 12dc4d87-ea95-4f68-ac86-b6b17d5448b5 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. - - {% if shuffled_order == 0 %} - - Sentence A: {{ sentence_good }} - - Sentence B: {{ sentence_bad }} - - {% else %} + {% set shuffled_order_options = [0, 1] | random %} - Sentence A: {{ sentence_bad }} - - Sentence B: {{ sentence_good }} - - {% endif %} - - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} - - {% endif %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: better_B_A - reference: '' - 1de92555-bf4a-40a6-8717-9f2fc3aaf5e0: !Template - answer_choices: 1 ||| 2 - id: 1de92555-bf4a-40a6-8717-9f2fc3aaf5e0 - jinja: '{% set shuffled_order = [0, 1] | random %} - - Which one of the following sentences is grammatical? Please answer 1 or 2. + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -76,13 +41,15 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: grammatical_which_one_A_B reference: '' - 1f1d88ad-d90f-42c7-b2b4-bd36b2fecae7: !Template + 1fcf9d58-92d1-4480-a4f3-01cc0421ff05: !Template answer_choices: A ||| B - id: 1f1d88ad-d90f-42c7-b2b4-bd36b2fecae7 + id: 1fcf9d58-92d1-4480-a4f3-01cc0421ff05 jinja: '{% set shuffled_order = [0, 1] | random %} + {% set shuffled_order_options = [0, 1] | random %} + Read the following two sentences. {% if shuffled_order == 0 %} @@ -99,8 +66,8 @@ templates: {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -116,28 +83,35 @@ templates: original_task: true name: better_A_B reference: '' - 1fd3ab83-7db4-4504-b26e-3450fccf8041: !Template + 50e1352f-74ff-40db-bf40-228a863405d3: !Template answer_choices: Yes ||| No - id: 1fd3ab83-7db4-4504-b26e-3450fccf8041 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + id: 50e1352f-74ff-40db-bf40-228a863405d3 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_good_no_yes + name: single_sentence_bad_yes_no reference: '' - 2a0675ad-6dc1-47ff-8456-09b6061973f1: !Template + b559f92f-12d3-4dd5-afe4-768a0bdb6aaf: !Template answer_choices: A ||| B - id: 2a0675ad-6dc1-47ff-8456-09b6061973f1 + id: b559f92f-12d3-4dd5-afe4-768a0bdb6aaf jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} @@ -171,12 +145,15 @@ templates: original_task: true name: grammatical_between_A_B reference: '' - 480140c6-d094-4111-9740-4cf92b62cbdc: !Template + d3dd8509-70fc-4937-aca6-3750274307ca: !Template answer_choices: 1 ||| 2 - id: 480140c6-d094-4111-9740-4cf92b62cbdc + id: d3dd8509-70fc-4937-aca6-3750274307ca jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} @@ -210,31 +187,19 @@ templates: original_task: true name: grammatical_between_1_2 reference: '' - 96a9b402-ab24-411c-9f7b-5b9df0ceb6c5: !Template + e214c1ce-5271-4d7b-99b2-70b31e4d6da0: !Template answer_choices: Yes ||| No - id: 96a9b402-ab24-411c-9f7b-5b9df0ceb6c5 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + id: e214c1ce-5271-4d7b-99b2-70b31e4d6da0 + jinja: '{% set shuffled_order = [0, 1] | random %} - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes - reference: '' - 99a676b2-2e8a-4e11-918f-09b1fe76253a: !Template - answer_choices: Yes ||| No - id: 99a676b2-2e8a-4e11-918f-09b1fe76253a - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: @@ -242,62 +207,38 @@ templates: original_task: false name: single_sentence_good_yes_no reference: '' - ad08cfc7-2ffa-48a9-b86b-fbbb5da87d90: !Template + f8ca733a-5639-4a48-b109-87e1c000ea86: !Template answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: ad08cfc7-2ffa-48a9-b86b-fbbb5da87d90 - jinja: '" ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: null_prompt - reference: '' - cc5a36ad-b6dd-4e18-8558-87d074b26e0d: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: cc5a36ad-b6dd-4e18-8558-87d074b26e0d + id: f8ca733a-5639-4a48-b109-87e1c000ea86 jinja: ' ||| {{ sentence_good }}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - Accuracy original_task: true - name: true_null_prompt - reference: '' - d6c43902-2bce-43c0-8129-936880506031: !Template - answer_choices: Yes ||| No - id: d6c43902-2bce-43c0-8129-936880506031 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no + name: null_prompt reference: '' - f90dbd74-7530-436f-b0f8-8f2b9658e858: !Template - answer_choices: A ||| B - id: f90dbd74-7530-436f-b0f8-8f2b9658e858 + fe09912e-a883-4b7d-b11f-9c65b8295a4e: !Template + answer_choices: 1 ||| 2 + id: fe09912e-a883-4b7d-b11f-9c65b8295a4e jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -317,5 +258,5 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: grammatical_which_one_1_2 reference: '' diff --git a/promptsource/templates/blimp/matrix_question_npi_licensor_present/templates.yaml b/promptsource/templates/blimp/matrix_question_npi_licensor_present/templates.yaml index 8fdb7c263..382f70bdc 100644 --- a/promptsource/templates/blimp/matrix_question_npi_licensor_present/templates.yaml +++ b/promptsource/templates/blimp/matrix_question_npi_licensor_present/templates.yaml @@ -1,23 +1,15 @@ dataset: blimp subset: matrix_question_npi_licensor_present templates: - 0c3e2da6-df0c-4bca-8ed0-e95cfa0d8f9d: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 0c3e2da6-df0c-4bca-8ed0-e95cfa0d8f9d - jinja: '" ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: null_prompt - reference: '' - 2cfa7c09-25ff-4e66-a07a-886d868a3ed3: !Template + 059851a0-7f9d-43e8-a131-bbdb88d2f68f: !Template answer_choices: A ||| B - id: 2cfa7c09-25ff-4e66-a07a-886d868a3ed3 + id: 059851a0-7f9d-43e8-a131-bbdb88d2f68f jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} @@ -51,51 +43,35 @@ templates: original_task: true name: grammatical_between_A_B reference: '' - 324780ec-3fa7-4b20-ba5d-a2cf8dbc2e0d: !Template - answer_choices: 1 ||| 2 - id: 324780ec-3fa7-4b20-ba5d-a2cf8dbc2e0d + 25a49a3a-0826-4b55-8638-f2168bcd80b9: !Template + answer_choices: Yes ||| No + id: 25a49a3a-0826-4b55-8638-f2168bcd80b9 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. - - {% if shuffled_order == 0 %} - - 1: {{ sentence_good }} - - 2: {{ sentence_bad }} - - {% else %} - - 1: {{ sentence_bad }} - - 2: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - {{ answer_choices[0] }} + {% set shuffled_order_options = [0, 1] | random %} - {% else %} + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" - {{ answer_choices[1] }} - - {% endif %}' + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: grammatical_between_1_2 + original_task: false + name: single_sentence_bad_yes_no reference: '' - 63c864fc-0e20-4548-99bf-498d6f55e491: !Template + 2ff72c8a-e156-4cf3-9522-9d9a338a3e50: !Template answer_choices: 1 ||| 2 - id: 63c864fc-0e20-4548-99bf-498d6f55e491 + id: 2ff72c8a-e156-4cf3-9522-9d9a338a3e50 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} @@ -127,63 +103,46 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 - reference: '' - 97d1289e-9ed4-41b1-9d67-4c9d24842fc4: !Template - answer_choices: Yes ||| No - id: 97d1289e-9ed4-41b1-9d67-4c9d24842fc4 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes + name: grammatical_between_1_2 reference: '' - 9a4e55cc-1252-4200-8510-0263b2343e44: !Template - answer_choices: Yes ||| No - id: 9a4e55cc-1252-4200-8510-0263b2343e44 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + a85cb8d7-76a9-4024-b63e-cfcfd00bea67: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: a85cb8d7-76a9-4024-b63e-cfcfd00bea67 + jinja: ' ||| {{ sentence_good }}' metadata: !TemplateMetadata - choices_in_prompt: true + choices_in_prompt: false metrics: - Accuracy - original_task: false - name: single_sentence_good_no_yes + original_task: true + name: null_prompt reference: '' - a8a277d4-1aa4-41dd-b243-085bee5595aa: !Template + af64a7ec-3c17-42f6-a183-9aa63c2c1d3e: !Template answer_choices: A ||| B - id: a8a277d4-1aa4-41dd-b243-085bee5595aa + id: af64a7ec-3c17-42f6-a183-9aa63c2c1d3e jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -197,13 +156,35 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: grammatical_which_one_A_B reference: '' - b7107e84-8173-40d5-aa22-8a477461381b: !Template + d0748d18-8341-4e1d-b462-2a05c6bf2996: !Template + answer_choices: Yes ||| No + id: d0748d18-8341-4e1d-b462-2a05c6bf2996 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + eab99bb5-b797-46d7-a075-eb416594cdc9: !Template answer_choices: A ||| B - id: b7107e84-8173-40d5-aa22-8a477461381b + id: eab99bb5-b797-46d7-a075-eb416594cdc9 jinja: '{% set shuffled_order = [0, 1] | random %} + {% set shuffled_order_options = [0, 1] | random %} + Read the following two sentences. {% if shuffled_order == 0 %} @@ -220,8 +201,8 @@ templates: {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -237,40 +218,27 @@ templates: original_task: true name: better_A_B reference: '' - ba7709d8-1f30-4c08-b059-c6ec0ebb68c8: !Template - answer_choices: Yes ||| No - id: ba7709d8-1f30-4c08-b059-c6ec0ebb68c8 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no - reference: '' - bb48ee5e-9c07-4cc7-a671-2984e1d4ef90: !Template - answer_choices: A ||| B - id: bb48ee5e-9c07-4cc7-a671-2984e1d4ef90 + ec11e437-3559-48bc-b066-abb789d43a98: !Template + answer_choices: 1 ||| 2 + id: ec11e437-3559-48bc-b066-abb789d43a98 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -290,32 +258,5 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B - reference: '' - f712b9af-fe4b-4927-a39b-c35d0ad2482d: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: f712b9af-fe4b-4927-a39b-c35d0ad2482d - jinja: ' ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: true_null_prompt - reference: '' - f9f2602a-6225-418e-8a51-9cbdec53daa0: !Template - answer_choices: Yes ||| No - id: f9f2602a-6225-418e-8a51-9cbdec53daa0 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no + name: grammatical_which_one_1_2 reference: '' diff --git a/promptsource/templates/blimp/npi_present_1/templates.yaml b/promptsource/templates/blimp/npi_present_1/templates.yaml index 9e88762db..2a9967746 100644 --- a/promptsource/templates/blimp/npi_present_1/templates.yaml +++ b/promptsource/templates/blimp/npi_present_1/templates.yaml @@ -1,50 +1,15 @@ dataset: blimp subset: npi_present_1 templates: - 0f05dfa3-8987-4158-bfbb-6e722cd4ac6b: !Template - answer_choices: Yes ||| No - id: 0f05dfa3-8987-4158-bfbb-6e722cd4ac6b - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no - reference: '' - 3430b62d-4b7d-4edf-bba1-8ee873c6d250: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 3430b62d-4b7d-4edf-bba1-8ee873c6d250 - jinja: ' ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: true_null_prompt - reference: '' - 3ab2d04c-7ccb-41ba-93d7-0a6b8d918248: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 3ab2d04c-7ccb-41ba-93d7-0a6b8d918248 - jinja: '" ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: null_prompt - reference: '' - 441d513b-b700-4949-bd4e-ed6d52867266: !Template + 39a7b62a-0043-4f86-99c4-8a74e7e6ed9d: !Template answer_choices: A ||| B - id: 441d513b-b700-4949-bd4e-ed6d52867266 + id: 39a7b62a-0043-4f86-99c4-8a74e7e6ed9d jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} @@ -76,14 +41,17 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: grammatical_which_one_A_B reference: '' - 53522fa9-e908-40af-bda2-fbb0af141047: !Template + 419aeb25-4a54-4572-bc52-46e8ae98dd5c: !Template answer_choices: 1 ||| 2 - id: 53522fa9-e908-40af-bda2-fbb0af141047 + id: 419aeb25-4a54-4572-bc52-46e8ae98dd5c jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} @@ -117,99 +85,44 @@ templates: original_task: true name: grammatical_between_1_2 reference: '' - 707984c7-d933-4a85-b5d1-e011321b41e9: !Template - answer_choices: A ||| B - id: 707984c7-d933-4a85-b5d1-e011321b41e9 - jinja: '{% set shuffled_order = [0, 1] | random %} - - Read the following two sentences. - - {% if shuffled_order == 0 %} - - Sentence A: {{ sentence_good }} - - Sentence B: {{ sentence_bad }} - - {% else %} - - Sentence A: {{ sentence_bad }} - - Sentence B: {{ sentence_good }} - - {% endif %} - - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} - - {% endif %}' + 6c628463-ca54-43ff-b847-ea73eb9184f7: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 6c628463-ca54-43ff-b847-ea73eb9184f7 + jinja: ' ||| {{ sentence_good }}' metadata: !TemplateMetadata - choices_in_prompt: true + choices_in_prompt: false metrics: - Accuracy original_task: true - name: better_A_B - reference: '' - 83c92e79-ee02-4bd6-8583-3dcdcf466e8a: !Template - answer_choices: Yes ||| No - id: 83c92e79-ee02-4bd6-8583-3dcdcf466e8a - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes - reference: '' - 94212f12-3828-4123-aa16-bfe82bf067e1: !Template - answer_choices: Yes ||| No - id: 94212f12-3828-4123-aa16-bfe82bf067e1 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes + name: null_prompt reference: '' - 9d489f52-223c-4ba5-9e48-7024245eefd6: !Template + 7f04187c-ac4e-474a-8e0e-d5ae69665383: !Template answer_choices: A ||| B - id: 9d489f52-223c-4ba5-9e48-7024245eefd6 + id: 7f04187c-ac4e-474a-8e0e-d5ae69665383 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -223,17 +136,21 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: grammatical_between_A_B reference: '' - acb76db2-c9f5-49fb-87b6-4f12a05e22c6: !Template + 8f87db45-1335-49f2-8d61-0cb245a33671: !Template answer_choices: Yes ||| No - id: acb76db2-c9f5-49fb-87b6-4f12a05e22c6 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + id: 8f87db45-1335-49f2-8d61-0cb245a33671 + jinja: '{% set shuffled_order = [0, 1] | random %} - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: @@ -241,12 +158,15 @@ templates: original_task: false name: single_sentence_good_yes_no reference: '' - bb82ebdd-bf2d-45f4-ba14-6efa4c5495d5: !Template + bd46ec4a-98b0-4775-8383-c0118161c9fa: !Template answer_choices: 1 ||| 2 - id: bb82ebdd-bf2d-45f4-ba14-6efa4c5495d5 + id: bd46ec4a-98b0-4775-8383-c0118161c9fa jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} @@ -280,30 +200,51 @@ templates: original_task: true name: grammatical_which_one_1_2 reference: '' - f679d29f-ed40-4bda-a640-c8fc125cd315: !Template + ed1b7e89-2e19-4b67-bb0d-6a9b3c1a3c1f: !Template + answer_choices: Yes ||| No + id: ed1b7e89-2e19-4b67-bb0d-6a9b3c1a3c1f + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + f30fa011-e77c-450e-8771-c10b10f2f367: !Template answer_choices: A ||| B - id: f679d29f-ed40-4bda-a640-c8fc125cd315 + id: f30fa011-e77c-450e-8771-c10b10f2f367 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -317,5 +258,5 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: better_A_B reference: '' diff --git a/promptsource/templates/blimp/npi_present_2/templates.yaml b/promptsource/templates/blimp/npi_present_2/templates.yaml index 2a50689f7..37e6695c0 100644 --- a/promptsource/templates/blimp/npi_present_2/templates.yaml +++ b/promptsource/templates/blimp/npi_present_2/templates.yaml @@ -1,51 +1,46 @@ dataset: blimp subset: npi_present_2 templates: - 189f7c26-e8ee-40df-9e2b-cd2438d2b8ca: !Template - answer_choices: A ||| B - id: 189f7c26-e8ee-40df-9e2b-cd2438d2b8ca + 1ef38606-ee86-42e3-acab-d8406b4d4124: !Template + answer_choices: Yes ||| No + id: 1ef38606-ee86-42e3-acab-d8406b4d4124 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. - - {% if shuffled_order == 0 %} - - A: {{ sentence_good }} - - B: {{ sentence_bad }} - - {% else %} + {% set shuffled_order_options = [0, 1] | random %} - A: {{ sentence_bad }} - - B: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} - - {% endif %}' + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 2157e41e-8ad7-466b-ba8d-49ac6fa544ed: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 2157e41e-8ad7-466b-ba8d-49ac6fa544ed + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy original_task: true - name: grammatical_which_one_A_B + name: null_prompt reference: '' - 18d8882d-7f23-420f-a321-d8b3a039ce86: !Template + 21bc7e99-6dff-46e0-8eb5-04256f170dfc: !Template answer_choices: 1 ||| 2 - id: 18d8882d-7f23-420f-a321-d8b3a039ce86 + id: 21bc7e99-6dff-46e0-8eb5-04256f170dfc jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} @@ -77,47 +72,35 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 - reference: '' - 1f9b7c05-c643-4434-8315-624b01b7af9f: !Template - answer_choices: Yes ||| No - id: 1f9b7c05-c643-4434-8315-624b01b7af9f - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no + name: grammatical_between_1_2 reference: '' - 2c6c8dcc-6c99-45e2-a393-db259665e794: !Template - answer_choices: A ||| B - id: 2c6c8dcc-6c99-45e2-a393-db259665e794 + 24d2dc86-e2f3-4537-8c6c-d9b119208140: !Template + answer_choices: 1 ||| 2 + id: 24d2dc86-e2f3-4537-8c6c-d9b119208140 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -131,47 +114,55 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: grammatical_which_one_1_2 reference: '' - 513f5f9f-10b5-4704-ac5e-781e30bb3e22: !Template + 63a36bd9-8f58-4c79-8965-c1b76d461849: !Template answer_choices: Yes ||| No - id: 513f5f9f-10b5-4704-ac5e-781e30bb3e22 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + id: 63a36bd9-8f58-4c79-8965-c1b76d461849 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_good_no_yes + name: single_sentence_good_yes_no reference: '' - 5b795bc2-82d5-442c-bee0-e542b1d647ae: !Template + b8864770-3d48-4499-a1b4-81fa66764a1c: !Template answer_choices: A ||| B - id: 5b795bc2-82d5-442c-bee0-e542b1d647ae + id: b8864770-3d48-4499-a1b4-81fa66764a1c jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -185,26 +176,29 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: grammatical_between_A_B reference: '' - 5d3d3930-eb49-4f0d-9e8c-6fa3ac8bc309: !Template - answer_choices: 1 ||| 2 - id: 5d3d3930-eb49-4f0d-9e8c-6fa3ac8bc309 + c5bc3e1e-f756-4bff-86e8-bfefd6eaa937: !Template + answer_choices: A ||| B + id: c5bc3e1e-f756-4bff-86e8-bfefd6eaa937 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -224,86 +218,33 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 - reference: '' - 89120a35-57b5-4654-b35d-3fa817d64b18: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 89120a35-57b5-4654-b35d-3fa817d64b18 - jinja: '" ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: null_prompt - reference: '' - b084e594-0a98-4745-add5-291afcecff39: !Template - answer_choices: Yes ||| No - id: b084e594-0a98-4745-add5-291afcecff39 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no - reference: '' - db6c97b6-2518-40d1-9e73-809b2a259cf4: !Template - answer_choices: Yes ||| No - id: db6c97b6-2518-40d1-9e73-809b2a259cf4 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes - reference: '' - dccf9f87-07ed-4efb-a54e-5e3883b802b6: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: dccf9f87-07ed-4efb-a54e-5e3883b802b6 - jinja: ' ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: true_null_prompt + name: grammatical_which_one_A_B reference: '' - e3854983-90a4-4b88-8628-7cd3652e638d: !Template + e93bdcbe-887b-4935-9e9d-61662829622a: !Template answer_choices: A ||| B - id: e3854983-90a4-4b88-8628-7cd3652e638d + id: e93bdcbe-887b-4935-9e9d-61662829622a jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -317,5 +258,5 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: better_A_B reference: '' diff --git a/promptsource/templates/blimp/only_npi_licensor_present/templates.yaml b/promptsource/templates/blimp/only_npi_licensor_present/templates.yaml index 43ed33289..5c42308f8 100644 --- a/promptsource/templates/blimp/only_npi_licensor_present/templates.yaml +++ b/promptsource/templates/blimp/only_npi_licensor_present/templates.yaml @@ -1,40 +1,44 @@ dataset: blimp subset: only_npi_licensor_present templates: - 2b35f684-b468-46db-977e-d2c0bfd3db00: !Template + 24fa314d-7c8f-49e2-a684-e5ad128e0e7d: !Template answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 2b35f684-b468-46db-977e-d2c0bfd3db00 + id: 24fa314d-7c8f-49e2-a684-e5ad128e0e7d jinja: ' ||| {{ sentence_good }}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - Accuracy original_task: true - name: true_null_prompt + name: null_prompt reference: '' - 2e45cdb6-9f51-45ff-9d80-5537b69d7ca6: !Template + 66871ba2-5e1a-4d07-a10e-793bfed083c2: !Template answer_choices: A ||| B - id: 2e45cdb6-9f51-45ff-9d80-5537b69d7ca6 + id: 66871ba2-5e1a-4d07-a10e-793bfed083c2 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -48,26 +52,69 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: grammatical_between_A_B reference: '' - 58893642-b4d2-498b-854d-475b05c18fd6: !Template - answer_choices: A ||| B - id: 58893642-b4d2-498b-854d-475b05c18fd6 + 6ff97bae-77eb-4619-940a-8c529d8c4d08: !Template + answer_choices: Yes ||| No + id: 6ff97bae-77eb-4619-940a-8c529d8c4d08 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 8e80598f-4497-4a5f-bbd4-e37f93460af6: !Template + answer_choices: Yes ||| No + id: 8e80598f-4497-4a5f-bbd4-e37f93460af6 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + a7db897c-bcea-4726-aaf4-07502d8cac5c: !Template + answer_choices: 1 ||| 2 + id: a7db897c-bcea-4726-aaf4-07502d8cac5c jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -87,14 +134,17 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: grammatical_between_1_2 reference: '' - 5ce28040-abd8-4e7f-a3a7-628550ed4604: !Template + b7a9dcd3-fd60-412d-b0eb-7dd7365c97ab: !Template answer_choices: A ||| B - id: 5ce28040-abd8-4e7f-a3a7-628550ed4604 + id: b7a9dcd3-fd60-412d-b0eb-7dd7365c97ab jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} @@ -126,14 +176,17 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: grammatical_which_one_A_B reference: '' - 6d77f04a-3602-4aad-9842-796008030b91: !Template + b8bf85da-cc09-4c2c-adbf-8013785c6e23: !Template answer_choices: 1 ||| 2 - id: 6d77f04a-3602-4aad-9842-796008030b91 + id: b8bf85da-cc09-4c2c-adbf-8013785c6e23 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} @@ -167,54 +220,13 @@ templates: original_task: true name: grammatical_which_one_1_2 reference: '' - 8215ed1e-f8cb-4b4e-af6f-3f618204ff0d: !Template - answer_choices: Yes ||| No - id: 8215ed1e-f8cb-4b4e-af6f-3f618204ff0d - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no - reference: '' - 8a622aa9-0dd1-44ac-aade-269b9e0848e0: !Template - answer_choices: Yes ||| No - id: 8a622aa9-0dd1-44ac-aade-269b9e0848e0 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes - reference: '' - 8e69a142-f645-4b24-8838-82b14a975335: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 8e69a142-f645-4b24-8838-82b14a975335 - jinja: '" ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: null_prompt - reference: '' - ab945f21-42f5-42d7-971b-5bc7f407ed87: !Template + c7b622be-51e2-4579-97b6-bf6500621aa4: !Template answer_choices: A ||| B - id: ab945f21-42f5-42d7-971b-5bc7f407ed87 + id: c7b622be-51e2-4579-97b6-bf6500621aa4 jinja: '{% set shuffled_order = [0, 1] | random %} + {% set shuffled_order_options = [0, 1] | random %} + Read the following two sentences. {% if shuffled_order == 0 %} @@ -231,8 +243,8 @@ templates: {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -248,74 +260,3 @@ templates: original_task: true name: better_A_B reference: '' - bba80804-0dd8-4448-9fb9-96d25dcd7bcf: !Template - answer_choices: Yes ||| No - id: bba80804-0dd8-4448-9fb9-96d25dcd7bcf - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no - reference: '' - ced631e8-585a-4baa-90c5-b06554a89e6a: !Template - answer_choices: Yes ||| No - id: ced631e8-585a-4baa-90c5-b06554a89e6a - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes - reference: '' - e9de0279-7efc-40ea-bf52-f58a4d0eb079: !Template - answer_choices: 1 ||| 2 - id: e9de0279-7efc-40ea-bf52-f58a4d0eb079 - jinja: '{% set shuffled_order = [0, 1] | random %} - - Between the two sentences, which one is grammatical? Please answer 1 or 2. - - {% if shuffled_order == 0 %} - - 1: {{ sentence_good }} - - 2: {{ sentence_bad }} - - {% else %} - - 1: {{ sentence_bad }} - - 2: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} - - {% endif %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: grammatical_between_1_2 - reference: '' diff --git a/promptsource/templates/blimp/only_npi_scope/templates.yaml b/promptsource/templates/blimp/only_npi_scope/templates.yaml index 1945eb750..caee2be2b 100644 --- a/promptsource/templates/blimp/only_npi_scope/templates.yaml +++ b/promptsource/templates/blimp/only_npi_scope/templates.yaml @@ -1,112 +1,71 @@ dataset: blimp subset: only_npi_scope templates: - 09c6c213-08ff-4e54-8651-8506dd1c1c23: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 09c6c213-08ff-4e54-8651-8506dd1c1c23 - jinja: ' ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: true_null_prompt - reference: '' - 0ac1af63-efc8-4320-ad84-19aecaf5e18b: !Template + 05e40054-b360-48a3-84c3-82da5de36bc7: !Template answer_choices: Yes ||| No - id: 0ac1af63-efc8-4320-ad84-19aecaf5e18b - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes - reference: '' - 25eae00b-576f-4e77-9869-3c1108d83e84: !Template - answer_choices: A ||| B - id: 25eae00b-576f-4e77-9869-3c1108d83e84 + id: 05e40054-b360-48a3-84c3-82da5de36bc7 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. - - {% if shuffled_order == 0 %} - - A: {{ sentence_good }} - - B: {{ sentence_bad }} - - {% else %} + {% set shuffled_order_options = [0, 1] | random %} - A: {{ sentence_bad }} - - B: {{ sentence_good }} - - {% endif %} + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" - ||| - - {% if shuffled_order == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} - - {% endif %}' + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: grammatical_between_A_B + original_task: false + name: single_sentence_bad_yes_no reference: '' - 26402983-4cc9-4f60-8876-6abb2e668af1: !Template + 169f2bdf-d96c-4614-a3c2-23063c5109d5: !Template answer_choices: Yes ||| No - id: 26402983-4cc9-4f60-8876-6abb2e668af1 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + id: 169f2bdf-d96c-4614-a3c2-23063c5109d5 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_good_no_yes + name: single_sentence_good_yes_no reference: '' - 2ec5cad4-5f63-4e8b-89f4-89fa23d77051: !Template - answer_choices: 1 ||| 2 - id: 2ec5cad4-5f63-4e8b-89f4-89fa23d77051 + 3511bc99-c76d-4779-8edd-db0e9d07279b: !Template + answer_choices: A ||| B + id: 3511bc99-c76d-4779-8edd-db0e9d07279b jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -120,30 +79,17 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 - reference: '' - 5c438c94-a594-40f9-8e37-0566363bde37: !Template - answer_choices: Yes ||| No - id: 5c438c94-a594-40f9-8e37-0566363bde37 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no + name: better_A_B reference: '' - 7b41f9cd-24a0-4436-b4eb-36a433ba72a1: !Template + 37ec526e-e774-4707-81c4-8f934c0f8e56: !Template answer_choices: A ||| B - id: 7b41f9cd-24a0-4436-b4eb-36a433ba72a1 + id: 37ec526e-e774-4707-81c4-8f934c0f8e56 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} @@ -175,31 +121,35 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: grammatical_between_A_B reference: '' - 85afff0b-e9d9-497f-b7d6-7ffa9a719a26: !Template - answer_choices: A ||| B - id: 85afff0b-e9d9-497f-b7d6-7ffa9a719a26 + 8cb778e7-9143-41b5-b5cd-4861e5fd541e: !Template + answer_choices: 1 ||| 2 + id: 8cb778e7-9143-41b5-b5cd-4861e5fd541e jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -213,42 +163,35 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B - reference: '' - af88d64d-85b6-4cc5-ba7b-4a6dc5d897a8: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: af88d64d-85b6-4cc5-ba7b-4a6dc5d897a8 - jinja: '" ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: null_prompt + name: grammatical_between_1_2 reference: '' - c6532a04-941c-4a74-8939-910a95b7338f: !Template - answer_choices: A ||| B - id: c6532a04-941c-4a74-8939-910a95b7338f + be758fd8-40e8-45d7-a5a7-a58ec468d540: !Template + answer_choices: 1 ||| 2 + id: be758fd8-40e8-45d7-a5a7-a58ec468d540 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -262,42 +205,29 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A - reference: '' - c8d407aa-e94d-4ac1-a7f5-8607de064195: !Template - answer_choices: Yes ||| No - id: c8d407aa-e94d-4ac1-a7f5-8607de064195 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no + name: grammatical_which_one_1_2 reference: '' - dfa910b3-6aa2-4534-afc6-6072fd30de20: !Template - answer_choices: 1 ||| 2 - id: dfa910b3-6aa2-4534-afc6-6072fd30de20 + c2f57af5-3e4c-4a3d-8ed3-48ff5bcd84e8: !Template + answer_choices: A ||| B + id: c2f57af5-3e4c-4a3d-8ed3-48ff5bcd84e8 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -317,5 +247,16 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: grammatical_which_one_A_B + reference: '' + dcaf199f-bf2e-4c60-ba9e-c542574c33cf: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: dcaf199f-bf2e-4c60-ba9e-c542574c33cf + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt reference: '' diff --git a/promptsource/templates/blimp/passive_1/templates.yaml b/promptsource/templates/blimp/passive_1/templates.yaml index 1c9110f38..5a2fa9bcc 100644 --- a/promptsource/templates/blimp/passive_1/templates.yaml +++ b/promptsource/templates/blimp/passive_1/templates.yaml @@ -1,12 +1,15 @@ dataset: blimp subset: passive_1 templates: - 25bd86e2-da2e-45b5-a7eb-ffbb4cb4e3c6: !Template + 04fda615-27b1-465e-9440-4c96b9b388fc: !Template answer_choices: 1 ||| 2 - id: 25bd86e2-da2e-45b5-a7eb-ffbb4cb4e3c6 + id: 04fda615-27b1-465e-9440-4c96b9b388fc jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} @@ -40,53 +43,19 @@ templates: original_task: true name: grammatical_between_1_2 reference: '' - 4cd96a94-68e3-4c21-88cd-3f5cab3f9b22: !Template - answer_choices: A ||| B - id: 4cd96a94-68e3-4c21-88cd-3f5cab3f9b22 + 0602ba7a-91b1-4054-940b-84f5873397c6: !Template + answer_choices: Yes ||| No + id: 0602ba7a-91b1-4054-940b-84f5873397c6 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. - - {% if shuffled_order == 0 %} - - Sentence A: {{ sentence_good }} - - Sentence B: {{ sentence_bad }} - - {% else %} - - Sentence A: {{ sentence_bad }} - - Sentence B: {{ sentence_good }} - - {% endif %} - - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} - - {{ answer_choices[0] }} - - {% else %} + {% set shuffled_order_options = [0, 1] | random %} - {{ answer_choices[1] }} + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" - {% endif %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: better_A_B - reference: '' - 63f89191-36ab-4e9e-ba46-ff2051b59879: !Template - answer_choices: Yes ||| No - id: 63f89191-36ab-4e9e-ba46-ff2051b59879 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: @@ -94,72 +63,33 @@ templates: original_task: false name: single_sentence_good_yes_no reference: '' - 7cf041bf-4063-46b0-88f1-dc06bbe2fe30: !Template - answer_choices: Yes ||| No - id: 7cf041bf-4063-46b0-88f1-dc06bbe2fe30 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no - reference: '' - 7da525bb-6e3c-4ae0-82d3-9080c7684999: !Template - answer_choices: Yes ||| No - id: 7da525bb-6e3c-4ae0-82d3-9080c7684999 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes - reference: '' - 883d09a8-64b7-4f97-853a-8aee97cf9d6d: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 883d09a8-64b7-4f97-853a-8aee97cf9d6d - jinja: '" ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: null_prompt - reference: '' - 9c6000cf-cab5-498a-a978-c34961b192a1: !Template + 44cdd60a-bd52-46d5-9c17-f1784ea50262: !Template answer_choices: A ||| B - id: 9c6000cf-cab5-498a-a978-c34961b192a1 + id: 44cdd60a-bd52-46d5-9c17-f1784ea50262 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -173,43 +103,44 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: grammatical_which_one_A_B reference: '' - ae4277b8-f24a-4fc3-bed2-0096cf43e375: !Template + 59071832-47f0-403f-9b84-8ee181a470ad: !Template answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: ae4277b8-f24a-4fc3-bed2-0096cf43e375 + id: 59071832-47f0-403f-9b84-8ee181a470ad jinja: ' ||| {{ sentence_good }}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - Accuracy original_task: true - name: true_null_prompt + name: null_prompt reference: '' - ba726648-94f2-43f0-9ded-513514570bc7: !Template - answer_choices: 1 ||| 2 - id: ba726648-94f2-43f0-9ded-513514570bc7 + 737871d0-ec0b-4295-bd8c-cc9a6c39e208: !Template + answer_choices: A ||| B + id: 737871d0-ec0b-4295-bd8c-cc9a6c39e208 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -223,30 +154,17 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 - reference: '' - c1de02ea-9280-4fef-902c-c9fc58c7e297: !Template - answer_choices: Yes ||| No - id: c1de02ea-9280-4fef-902c-c9fc58c7e297 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes + name: better_A_B reference: '' - f2f4d471-a4ae-4b2a-8dbf-377720cb6e2e: !Template + 7fd8760e-3c98-49d3-b508-a5547fbb18aa: !Template answer_choices: A ||| B - id: f2f4d471-a4ae-4b2a-8dbf-377720cb6e2e + id: 7fd8760e-3c98-49d3-b508-a5547fbb18aa jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} @@ -280,24 +198,27 @@ templates: original_task: true name: grammatical_between_A_B reference: '' - f42993b2-eefd-4413-b786-01f98342a66f: !Template - answer_choices: A ||| B - id: f42993b2-eefd-4413-b786-01f98342a66f + b0d0957e-e73a-4019-ad55-8165f918c5a0: !Template + answer_choices: 1 ||| 2 + id: b0d0957e-e73a-4019-ad55-8165f918c5a0 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -317,5 +238,25 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: grammatical_which_one_1_2 + reference: '' + d064c4af-e701-43a8-9bff-5699dc39d8f8: !Template + answer_choices: Yes ||| No + id: d064c4af-e701-43a8-9bff-5699dc39d8f8 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no reference: '' diff --git a/promptsource/templates/blimp/passive_2/templates.yaml b/promptsource/templates/blimp/passive_2/templates.yaml index 799d1a159..e4225391e 100644 --- a/promptsource/templates/blimp/passive_2/templates.yaml +++ b/promptsource/templates/blimp/passive_2/templates.yaml @@ -1,35 +1,27 @@ dataset: blimp subset: passive_2 templates: - 23f63696-2ef8-4b22-8498-8c4a281037f0: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 23f63696-2ef8-4b22-8498-8c4a281037f0 - jinja: ' ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: true_null_prompt - reference: '' - 3e13c0f4-d863-4c4c-9551-6d391d78999b: !Template - answer_choices: A ||| B - id: 3e13c0f4-d863-4c4c-9551-6d391d78999b + 06b9fe19-f65a-4058-b92e-e3bde058d82e: !Template + answer_choices: 1 ||| 2 + id: 06b9fe19-f65a-4058-b92e-e3bde058d82e jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -49,70 +41,53 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: grammatical_which_one_1_2 reference: '' - 4203849e-c999-428e-9dc8-45616735d6da: !Template - answer_choices: A ||| B - id: 4203849e-c999-428e-9dc8-45616735d6da + 2c585c96-1c50-4695-8aef-d6abfb42dcd7: !Template + answer_choices: Yes ||| No + id: 2c585c96-1c50-4695-8aef-d6abfb42dcd7 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. - - {% if shuffled_order == 0 %} - - Sentence A: {{ sentence_good }} - - Sentence B: {{ sentence_bad }} - - {% else %} - - Sentence A: {{ sentence_bad }} - - Sentence B: {{ sentence_good }} - - {% endif %} + {% set shuffled_order_options = [0, 1] | random %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} - - {{ answer_choices[0] }} - - {% else %} + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" - {{ answer_choices[1] }} - - {% endif %}' + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: better_B_A + original_task: false + name: single_sentence_good_yes_no reference: '' - 4807d853-45bb-4baa-b4d3-21253fa651c6: !Template + 7c20c5d7-a548-4644-8a0f-5e926105b8b7: !Template answer_choices: A ||| B - id: 4807d853-45bb-4baa-b4d3-21253fa651c6 + id: 7c20c5d7-a548-4644-8a0f-5e926105b8b7 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -126,30 +101,17 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B - reference: '' - 5f9f8395-3c0b-40e7-a860-2c7cbb776645: !Template - answer_choices: Yes ||| No - id: 5f9f8395-3c0b-40e7-a860-2c7cbb776645 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes + name: better_A_B reference: '' - 8029a698-b8d8-425c-b7e5-7fa02df6d5b0: !Template + b973a531-bfbb-40fd-abc2-1d68ae7e91d3: !Template answer_choices: 1 ||| 2 - id: 8029a698-b8d8-425c-b7e5-7fa02df6d5b0 + id: b973a531-bfbb-40fd-abc2-1d68ae7e91d3 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} @@ -183,56 +145,38 @@ templates: original_task: true name: grammatical_between_1_2 reference: '' - 82c53b24-7029-44bc-9215-db9a4b9e3240: !Template - answer_choices: Yes ||| No - id: 82c53b24-7029-44bc-9215-db9a4b9e3240 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no - reference: '' - 9fbfb530-eb17-437a-b8ef-6738e5c4246e: !Template - answer_choices: Yes ||| No - id: 9fbfb530-eb17-437a-b8ef-6738e5c4246e - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' + c6a996bf-1528-4276-bc92-d8655e2d2449: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: c6a996bf-1528-4276-bc92-d8655e2d2449 + jinja: ' ||| {{ sentence_good }}' metadata: !TemplateMetadata - choices_in_prompt: true + choices_in_prompt: false metrics: - Accuracy - original_task: false - name: single_sentence_bad_yes_no + original_task: true + name: null_prompt reference: '' - b36882ca-2b6b-43e3-86b9-0acdb00d81a8: !Template - answer_choices: 1 ||| 2 - id: b36882ca-2b6b-43e3-86b9-0acdb00d81a8 + d1463fda-d34e-40b6-bca3-e6322145f5fd: !Template + answer_choices: A ||| B + id: d1463fda-d34e-40b6-bca3-e6322145f5fd jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -252,31 +196,35 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: grammatical_which_one_A_B reference: '' - dcfed41f-b7d2-49c6-90d8-5b6de7043126: !Template + d47cdd9f-f846-4731-bb25-ab51ead0fa91: !Template answer_choices: A ||| B - id: dcfed41f-b7d2-49c6-90d8-5b6de7043126 + id: d47cdd9f-f846-4731-bb25-ab51ead0fa91 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -290,32 +238,25 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: grammatical_between_A_B reference: '' - e045dd82-d3e8-4406-955e-f4a8eb0503b2: !Template + d95f71d4-9c2b-4539-9710-3121834b56cb: !Template answer_choices: Yes ||| No - id: e045dd82-d3e8-4406-955e-f4a8eb0503b2 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + id: d95f71d4-9c2b-4539-9710-3121834b56cb + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_bad_no_yes - reference: '' - f82801b8-7b3d-4b2e-b5c0-2eb0fbded769: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: f82801b8-7b3d-4b2e-b5c0-2eb0fbded769 - jinja: '" ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: null_prompt + name: single_sentence_bad_yes_no reference: '' diff --git a/promptsource/templates/blimp/principle_A_c_command/templates.yaml b/promptsource/templates/blimp/principle_A_c_command/templates.yaml index feed47741..8374e386a 100644 --- a/promptsource/templates/blimp/principle_A_c_command/templates.yaml +++ b/promptsource/templates/blimp/principle_A_c_command/templates.yaml @@ -1,39 +1,15 @@ dataset: blimp subset: principle_A_c_command templates: - 14df57be-0b56-4839-b1c7-7dd1ae33be77: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 14df57be-0b56-4839-b1c7-7dd1ae33be77 - jinja: ' ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: true_null_prompt - reference: '' - 17b77f2b-6a64-4617-aa7f-fa346bdfe2da: !Template - answer_choices: Yes ||| No - id: 17b77f2b-6a64-4617-aa7f-fa346bdfe2da - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no - reference: '' - 224440bb-ad58-44b8-b7e4-9fa911f039d3: !Template + 2b0ef285-4e29-4c9c-bd17-a0e7942b007d: !Template answer_choices: 1 ||| 2 - id: 224440bb-ad58-44b8-b7e4-9fa911f039d3 + id: 2b0ef285-4e29-4c9c-bd17-a0e7942b007d jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} @@ -65,13 +41,35 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: grammatical_which_one_1_2 reference: '' - 3629590c-1902-4af2-806f-e19ef74bf8fd: !Template + 4062dcf3-d7fe-4fd9-841c-1aec645cfa47: !Template + answer_choices: Yes ||| No + id: 4062dcf3-d7fe-4fd9-841c-1aec645cfa47 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 642f56eb-0e53-4cc0-96b4-4a5460c89c43: !Template answer_choices: A ||| B - id: 3629590c-1902-4af2-806f-e19ef74bf8fd + id: 642f56eb-0e53-4cc0-96b4-4a5460c89c43 jinja: '{% set shuffled_order = [0, 1] | random %} + {% set shuffled_order_options = [0, 1] | random %} + Read the following two sentences. {% if shuffled_order == 0 %} @@ -88,8 +86,8 @@ templates: {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -105,51 +103,27 @@ templates: original_task: true name: better_A_B reference: '' - 37b856ed-193d-4add-ac0e-0d5315c0e041: !Template - answer_choices: Yes ||| No - id: 37b856ed-193d-4add-ac0e-0d5315c0e041 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no - reference: '' - 6b0c2c8a-373b-4386-a642-b2c208701438: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 6b0c2c8a-373b-4386-a642-b2c208701438 - jinja: '" ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: null_prompt - reference: '' - 94d5681b-fb53-4fd0-9c56-b83806f5beaf: !Template - answer_choices: 1 ||| 2 - id: 94d5681b-fb53-4fd0-9c56-b83806f5beaf + 77857193-d7f8-41b1-bae9-3ea8c1b8a4e4: !Template + answer_choices: A ||| B + id: 77857193-d7f8-41b1-bae9-3ea8c1b8a4e4 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -169,63 +143,35 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 - reference: '' - a44f194a-4937-4425-8164-089dfd4c7c1d: !Template - answer_choices: Yes ||| No - id: a44f194a-4937-4425-8164-089dfd4c7c1d - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes - reference: '' - b78c89c0-72dd-4cdc-939f-63a83d7d83f5: !Template - answer_choices: Yes ||| No - id: b78c89c0-72dd-4cdc-939f-63a83d7d83f5 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes + name: grammatical_between_A_B reference: '' - bf775f9e-0e45-4365-b700-f9c6a4c1ab55: !Template - answer_choices: A ||| B - id: bf775f9e-0e45-4365-b700-f9c6a4c1ab55 + 78b70ffb-a7c0-4196-810d-0b7886172c28: !Template + answer_choices: 1 ||| 2 + id: 78b70ffb-a7c0-4196-810d-0b7886172c28 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -239,53 +185,37 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: grammatical_between_1_2 reference: '' - c34cc39d-7e16-4151-ba3b-ed09ea2beecc: !Template - answer_choices: A ||| B - id: c34cc39d-7e16-4151-ba3b-ed09ea2beecc + 7f543e11-9bbc-4175-9fcc-2cdeefd024bf: !Template + answer_choices: Yes ||| No + id: 7f543e11-9bbc-4175-9fcc-2cdeefd024bf jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. - - {% if shuffled_order == 0 %} - - A: {{ sentence_good }} - - B: {{ sentence_bad }} - - {% else %} + {% set shuffled_order_options = [0, 1] | random %} - A: {{ sentence_bad }} - - B: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" - {% endif %}' + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: grammatical_which_one_A_B + original_task: false + name: single_sentence_good_yes_no reference: '' - c7cbe94c-5cd1-4a9a-a3a7-ef7739705c92: !Template + b56211bb-d767-4494-a0b4-77c66f0c67b4: !Template answer_choices: A ||| B - id: c7cbe94c-5cd1-4a9a-a3a7-ef7739705c92 + id: b56211bb-d767-4494-a0b4-77c66f0c67b4 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} @@ -317,5 +247,16 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: grammatical_which_one_A_B + reference: '' + de3b026d-d470-4074-9271-39b04e28f29a: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: de3b026d-d470-4074-9271-39b04e28f29a + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt reference: '' diff --git a/promptsource/templates/blimp/principle_A_case_1/templates.yaml b/promptsource/templates/blimp/principle_A_case_1/templates.yaml index a1c35e1b8..4f057672e 100644 --- a/promptsource/templates/blimp/principle_A_case_1/templates.yaml +++ b/promptsource/templates/blimp/principle_A_case_1/templates.yaml @@ -1,12 +1,15 @@ dataset: blimp subset: principle_A_case_1 templates: - 020e3f46-809a-4f54-9a1b-aaebeb10b20a: !Template + 3779766b-d75e-4a0c-8019-829046811ed5: !Template answer_choices: 1 ||| 2 - id: 020e3f46-809a-4f54-9a1b-aaebeb10b20a + id: 3779766b-d75e-4a0c-8019-829046811ed5 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} @@ -40,54 +43,50 @@ templates: original_task: true name: grammatical_which_one_1_2 reference: '' - 0b92fd61-691b-448b-8d30-ba1a6f64209b: !Template - answer_choices: A ||| B - id: 0b92fd61-691b-448b-8d30-ba1a6f64209b + 49d9d4c4-0be1-4bf8-be40-42d05dd6829b: !Template + answer_choices: Yes ||| No + id: 49d9d4c4-0be1-4bf8-be40-42d05dd6829b jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. - - {% if shuffled_order == 0 %} - - A: {{ sentence_good }} - - B: {{ sentence_bad }} - - {% else %} - - A: {{ sentence_bad }} - - B: {{ sentence_good }} - - {% endif %} - - ||| + {% set shuffled_order_options = [0, 1] | random %} - {% if shuffled_order == 0 %} + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} - - {% endif %}' + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 7450e2e8-9bbd-486d-b857-7602012f8600: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 7450e2e8-9bbd-486d-b857-7602012f8600 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy original_task: true - name: grammatical_which_one_A_B + name: null_prompt reference: '' - 4639d962-0757-436f-b00c-1d76e78ba1c8: !Template + 8939f24b-7a3e-4a91-bda2-c428c2ca758c: !Template answer_choices: Yes ||| No - id: 4639d962-0757-436f-b00c-1d76e78ba1c8 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + id: 8939f24b-7a3e-4a91-bda2-c428c2ca758c + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: @@ -95,29 +94,33 @@ templates: original_task: false name: single_sentence_good_yes_no reference: '' - 4868f71e-5558-48b2-ace0-551b620c7164: !Template - answer_choices: A ||| B - id: 4868f71e-5558-48b2-ace0-551b620c7164 + 9675ffe4-e45b-413b-abb2-99d0af78ab29: !Template + answer_choices: 1 ||| 2 + id: 9675ffe4-e45b-413b-abb2-99d0af78ab29 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -131,48 +134,33 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B - reference: '' - 5a0db8b2-5e16-4df9-a390-a2b495762ba4: !Template - answer_choices: Yes ||| No - id: 5a0db8b2-5e16-4df9-a390-a2b495762ba4 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes + name: grammatical_between_1_2 reference: '' - 62f2bf9c-15af-42d7-abeb-816d4b7e984c: !Template + aed1b5f9-a572-4174-84aa-0d298753bd41: !Template answer_choices: A ||| B - id: 62f2bf9c-15af-42d7-abeb-816d4b7e984c + id: aed1b5f9-a572-4174-84aa-0d298753bd41 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -186,26 +174,29 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: better_A_B reference: '' - 7c697da0-4f1d-407f-9d7b-31417aac1186: !Template - answer_choices: 1 ||| 2 - id: 7c697da0-4f1d-407f-9d7b-31417aac1186 + c64a3a1e-289a-42ac-9979-1bd844da1b5d: !Template + answer_choices: A ||| B + id: c64a3a1e-289a-42ac-9979-1bd844da1b5d jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -225,69 +216,35 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 - reference: '' - 8292012a-5704-4343-97cb-83bb02f628ff: !Template - answer_choices: Yes ||| No - id: 8292012a-5704-4343-97cb-83bb02f628ff - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes - reference: '' - 9ec45076-a10c-462b-96e5-b5c43012f16c: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 9ec45076-a10c-462b-96e5-b5c43012f16c - jinja: ' ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: true_null_prompt - reference: '' - b1bac90e-bfd2-4cc5-8f5e-0677e2460abc: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: b1bac90e-bfd2-4cc5-8f5e-0677e2460abc - jinja: '" ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: null_prompt + name: grammatical_which_one_A_B reference: '' - fbfcc0fb-2955-4d87-9ace-0bd752a71378: !Template + e15f0627-2b98-4bde-947d-a4150428c510: !Template answer_choices: A ||| B - id: fbfcc0fb-2955-4d87-9ace-0bd752a71378 + id: e15f0627-2b98-4bde-947d-a4150428c510 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -301,21 +258,5 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A - reference: '' - fe227836-7a7e-41af-8fa8-6505a70eb3b9: !Template - answer_choices: Yes ||| No - id: fe227836-7a7e-41af-8fa8-6505a70eb3b9 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no + name: grammatical_between_A_B reference: '' diff --git a/promptsource/templates/blimp/principle_A_case_2/templates.yaml b/promptsource/templates/blimp/principle_A_case_2/templates.yaml index 00621d338..804cc90d5 100644 --- a/promptsource/templates/blimp/principle_A_case_2/templates.yaml +++ b/promptsource/templates/blimp/principle_A_case_2/templates.yaml @@ -1,43 +1,13 @@ dataset: blimp subset: principle_A_case_2 templates: - 1b97a778-bb48-4ac1-bc2c-606a14bda5a5: !Template - answer_choices: Yes ||| No - id: 1b97a778-bb48-4ac1-bc2c-606a14bda5a5 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no - reference: '' - 2111ed47-d96c-46e1-a88d-87e41fcaaf0c: !Template - answer_choices: Yes ||| No - id: 2111ed47-d96c-46e1-a88d-87e41fcaaf0c - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes - reference: '' - 21f07345-80de-4366-86d4-7cfa00c2c951: !Template + 005dc559-848e-4fc5-9798-45bb9542f4b9: !Template answer_choices: A ||| B - id: 21f07345-80de-4366-86d4-7cfa00c2c951 + id: 005dc559-848e-4fc5-9798-45bb9542f4b9 jinja: '{% set shuffled_order = [0, 1] | random %} + {% set shuffled_order_options = [0, 1] | random %} + Read the following two sentences. {% if shuffled_order == 0 %} @@ -54,8 +24,8 @@ templates: {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -69,26 +39,29 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: better_A_B reference: '' - 2217a186-ab74-42d7-bd74-4c17d3cda486: !Template - answer_choices: 1 ||| 2 - id: 2217a186-ab74-42d7-bd74-4c17d3cda486 + 01172621-67c3-4bfa-aae8-0580d3b6b125: !Template + answer_choices: A ||| B + id: 01172621-67c3-4bfa-aae8-0580d3b6b125 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -108,41 +81,37 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: grammatical_between_A_B reference: '' - 33d2aaa7-0ee4-4afc-b383-8cba8d34f2b6: !Template + 2d12c46e-58f0-4353-b6ae-bd9b773b4c18: !Template answer_choices: Yes ||| No - id: 33d2aaa7-0ee4-4afc-b383-8cba8d34f2b6 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + id: 2d12c46e-58f0-4353-b6ae-bd9b773b4c18 + jinja: '{% set shuffled_order = [0, 1] | random %} - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_good_no_yes - reference: '' - 3d95a90a-ed9e-4d7e-b486-fc6cbaf8c317: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 3d95a90a-ed9e-4d7e-b486-fc6cbaf8c317 - jinja: '" ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: null_prompt + name: single_sentence_bad_yes_no reference: '' - 3dc3e400-c463-4d49-96f2-cf863cf459de: !Template + 540ee3d2-f789-42b3-8614-1d867342159d: !Template answer_choices: 1 ||| 2 - id: 3dc3e400-c463-4d49-96f2-cf863cf459de + id: 540ee3d2-f789-42b3-8614-1d867342159d jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} @@ -174,26 +143,29 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: grammatical_which_one_1_2 reference: '' - 54811f76-307b-4685-a3c3-f747f6dfd7a3: !Template - answer_choices: A ||| B - id: 54811f76-307b-4685-a3c3-f747f6dfd7a3 + 69098dea-4e33-401b-b61a-82a25f551625: !Template + answer_choices: 1 ||| 2 + id: 69098dea-4e33-401b-b61a-82a25f551625 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -213,31 +185,35 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: grammatical_between_1_2 reference: '' - bea83acb-6c7c-499e-b51d-866db4df97f8: !Template + 7ed4158d-5939-4b71-8579-a836c4e9158c: !Template answer_choices: A ||| B - id: bea83acb-6c7c-499e-b51d-866db4df97f8 + id: 7ed4158d-5939-4b71-8579-a836c4e9158c jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -251,71 +227,36 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: grammatical_which_one_A_B reference: '' - cc388e7e-bc66-43e0-b964-8be7ceff81fa: !Template + a065510f-7097-45de-be3e-0b3f78b887e2: !Template answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: cc388e7e-bc66-43e0-b964-8be7ceff81fa + id: a065510f-7097-45de-be3e-0b3f78b887e2 jinja: ' ||| {{ sentence_good }}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - Accuracy original_task: true - name: true_null_prompt + name: null_prompt reference: '' - d5bdbca8-6eb9-4bb7-90af-254e3eb29cf4: !Template + df6ebc86-44be-4a68-9c1b-4a1246099eb8: !Template answer_choices: Yes ||| No - id: d5bdbca8-6eb9-4bb7-90af-254e3eb29cf4 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no - reference: '' - e05d659e-9c31-4e16-af3b-76d90b249f65: !Template - answer_choices: A ||| B - id: e05d659e-9c31-4e16-af3b-76d90b249f65 + id: df6ebc86-44be-4a68-9c1b-4a1246099eb8 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} - {% if shuffled_order == 0 %} - - A: {{ sentence_good }} - - B: {{ sentence_bad }} - - {% else %} - - A: {{ sentence_bad }} - - B: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - {{ answer_choices[0] }} - - {% else %} + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" - {{ answer_choices[1] }} - - {% endif %}' + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: grammatical_which_one_A_B + original_task: false + name: single_sentence_good_yes_no reference: '' diff --git a/promptsource/templates/blimp/principle_A_domain_1/templates.yaml b/promptsource/templates/blimp/principle_A_domain_1/templates.yaml index ae3e998f7..6a4b1e49f 100644 --- a/promptsource/templates/blimp/principle_A_domain_1/templates.yaml +++ b/promptsource/templates/blimp/principle_A_domain_1/templates.yaml @@ -1,68 +1,34 @@ dataset: blimp subset: principle_A_domain_1 templates: - 1189a014-98f3-4ade-b109-54b8d1f848d7: !Template + 2bbd2f4c-9c2b-41e0-b773-043baca30250: !Template answer_choices: A ||| B - id: 1189a014-98f3-4ade-b109-54b8d1f848d7 + id: 2bbd2f4c-9c2b-41e0-b773-043baca30250 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} - - {% endif %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: better_A_B - reference: '' - 20639897-58c5-490d-9267-f151776ac38c: !Template - answer_choices: A ||| B - id: 20639897-58c5-490d-9267-f151776ac38c - jinja: '{% set shuffled_order = [0, 1] | random %} - - Read the following two sentences. + ||| {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} - - Sentence B: {{ sentence_bad }} - - {% else %} - - Sentence A: {{ sentence_bad }} - - Sentence B: {{ sentence_good }} - - {% endif %} - - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} - {{ answer_choices[0] }} {% else %} @@ -75,28 +41,21 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A - reference: '' - a5fa2d01-a62a-4adf-b59a-cf5e11e62557: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: a5fa2d01-a62a-4adf-b59a-cf5e11e62557 - jinja: ' ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: true_null_prompt + name: grammatical_between_A_B reference: '' - b8637d62-51cd-4042-8bc6-39327c4d8b77: !Template + 3d340473-b40b-4002-91b1-10139368d878: !Template answer_choices: Yes ||| No - id: b8637d62-51cd-4042-8bc6-39327c4d8b77 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + id: 3d340473-b40b-4002-91b1-10139368d878 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: @@ -104,44 +63,35 @@ templates: original_task: false name: single_sentence_good_yes_no reference: '' - c9292620-ea8a-4df6-94c5-c41965971393: !Template + 469288fd-bede-42ea-b2e8-de48dd45a444: !Template answer_choices: Yes ||| No - id: c9292620-ea8a-4df6-94c5-c41965971393 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + id: 469288fd-bede-42ea-b2e8-de48dd45a444 + jinja: '{% set shuffled_order = [0, 1] | random %} - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes - reference: '' - ccc246e2-3c4a-474f-a24d-e5cb9851998a: !Template - answer_choices: Yes ||| No - id: ccc246e2-3c4a-474f-a24d-e5cb9851998a - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + {% set shuffled_order_options = [0, 1] | random %} - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_good_no_yes + name: single_sentence_bad_yes_no reference: '' - d19145e1-cd40-41d6-9f65-08a9cbd25595: !Template + 4e445447-cfd4-44e0-9a28-e82766410da7: !Template answer_choices: A ||| B - id: d19145e1-cd40-41d6-9f65-08a9cbd25595 + id: 4e445447-cfd4-44e0-9a28-e82766410da7 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} @@ -173,37 +123,29 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B - reference: '' - da416f81-40e7-4f21-a86a-b72587fdc375: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: da416f81-40e7-4f21-a86a-b72587fdc375 - jinja: '" ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: null_prompt + name: grammatical_which_one_A_B reference: '' - daa889ef-9eec-451c-ae3b-9c1a19b08399: !Template - answer_choices: A ||| B - id: daa889ef-9eec-451c-ae3b-9c1a19b08399 + 78b2287a-4a15-4e7a-9407-4fdcac441979: !Template + answer_choices: 1 ||| 2 + id: 78b2287a-4a15-4e7a-9407-4fdcac441979 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -223,14 +165,17 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: grammatical_which_one_1_2 reference: '' - dd7e3e25-08d2-4f46-ba4b-bd0689bbb8c9: !Template + 86b9e874-8e9f-407d-8a9e-1ca59c793a2d: !Template answer_choices: 1 ||| 2 - id: dd7e3e25-08d2-4f46-ba4b-bd0689bbb8c9 + id: 86b9e874-8e9f-407d-8a9e-1ca59c793a2d jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} @@ -264,30 +209,42 @@ templates: original_task: true name: grammatical_between_1_2 reference: '' - e1e5ea8d-4fda-4a50-bfe9-1939c062200b: !Template - answer_choices: 1 ||| 2 - id: e1e5ea8d-4fda-4a50-bfe9-1939c062200b + cf260efd-b607-4b6b-9089-972c5f680d7f: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: cf260efd-b607-4b6b-9089-972c5f680d7f + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + d7d066d2-6480-4589-a1a4-e39871d2bfda: !Template + answer_choices: A ||| B + id: d7d066d2-6480-4589-a1a4-e39871d2bfda jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -301,21 +258,5 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 - reference: '' - f07bbec3-f3f4-4328-ad11-0e0f55570c5b: !Template - answer_choices: Yes ||| No - id: f07bbec3-f3f4-4328-ad11-0e0f55570c5b - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no + name: better_A_B reference: '' diff --git a/promptsource/templates/blimp/principle_A_domain_2/templates.yaml b/promptsource/templates/blimp/principle_A_domain_2/templates.yaml index 5107135d0..923191944 100644 --- a/promptsource/templates/blimp/principle_A_domain_2/templates.yaml +++ b/promptsource/templates/blimp/principle_A_domain_2/templates.yaml @@ -1,46 +1,51 @@ dataset: blimp subset: principle_A_domain_2 templates: - 17efc25a-f1d3-4c30-9808-b53e024866fc: !Template + 0bc826f5-d7b1-4f6d-b697-ef6ad2c1200c: !Template answer_choices: Yes ||| No - id: 17efc25a-f1d3-4c30-9808-b53e024866fc - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + id: 0bc826f5-d7b1-4f6d-b697-ef6ad2c1200c + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_bad_no_yes + name: single_sentence_bad_yes_no reference: '' - 1cf30ce6-2854-4dd6-ae35-a4d7d172b056: !Template - answer_choices: 1 ||| 2 - id: 1cf30ce6-2854-4dd6-ae35-a4d7d172b056 + 1b27d629-a2fc-4269-9b5f-80e549ee60fa: !Template + answer_choices: A ||| B + id: 1b27d629-a2fc-4269-9b5f-80e549ee60fa jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -54,26 +59,29 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: better_A_B reference: '' - 39999513-7266-47a4-be66-7eb3b2ca4d18: !Template - answer_choices: 1 ||| 2 - id: 39999513-7266-47a4-be66-7eb3b2ca4d18 + 1ca4027f-6b05-4991-8008-f6ac0dde727f: !Template + answer_choices: A ||| B + id: 1ca4027f-6b05-4991-8008-f6ac0dde727f jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -93,96 +101,29 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 - reference: '' - 521b74bd-8322-4e95-85df-41ccce4c92f8: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 521b74bd-8322-4e95-85df-41ccce4c92f8 - jinja: ' ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: true_null_prompt - reference: '' - 56c62ec2-f6d6-480d-a7d4-79d81eb155b2: !Template - answer_choices: Yes ||| No - id: 56c62ec2-f6d6-480d-a7d4-79d81eb155b2 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no - reference: '' - 8ac81655-6870-407b-a04c-d2a74c0c2ef4: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 8ac81655-6870-407b-a04c-d2a74c0c2ef4 - jinja: '" ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: null_prompt - reference: '' - 92aeb196-9316-4094-99eb-ac6e9dff998f: !Template - answer_choices: Yes ||| No - id: 92aeb196-9316-4094-99eb-ac6e9dff998f - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no - reference: '' - a33a14dd-d063-40af-9c44-c80aa7ad995f: !Template - answer_choices: Yes ||| No - id: a33a14dd-d063-40af-9c44-c80aa7ad995f - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes + name: grammatical_which_one_A_B reference: '' - af3910c8-2853-4604-b0fe-916334c67762: !Template - answer_choices: A ||| B - id: af3910c8-2853-4604-b0fe-916334c67762 + 4cb7ba89-4eab-4095-9174-887353bc2986: !Template + answer_choices: 1 ||| 2 + id: 4cb7ba89-4eab-4095-9174-887353bc2986 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -202,14 +143,17 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: grammatical_between_1_2 reference: '' - b08ef95e-cf26-41b4-9656-51a4a1789e5c: !Template + 69defef5-befc-450c-b400-3471dac4be8d: !Template answer_choices: A ||| B - id: b08ef95e-cf26-41b4-9656-51a4a1789e5c + id: 69defef5-befc-450c-b400-3471dac4be8d jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} @@ -241,31 +185,35 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: grammatical_between_A_B reference: '' - cd6fa422-0f37-4049-83e2-f471b70e4b18: !Template - answer_choices: A ||| B - id: cd6fa422-0f37-4049-83e2-f471b70e4b18 + b26d04cb-8633-4e6e-a89e-a85a6dce1736: !Template + answer_choices: 1 ||| 2 + id: b26d04cb-8633-4e6e-a89e-a85a6dce1736 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -279,43 +227,36 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: grammatical_which_one_1_2 reference: '' - e3495bf0-f8a4-488b-925f-1a0abdde5d6d: !Template - answer_choices: A ||| B - id: e3495bf0-f8a4-488b-925f-1a0abdde5d6d + bb641f36-41e6-418b-9cdb-5b04ebf51715: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: bb641f36-41e6-418b-9cdb-5b04ebf51715 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + fe3c752d-dbbb-43fa-90a2-a866b30ea6dc: !Template + answer_choices: Yes ||| No + id: fe3c752d-dbbb-43fa-90a2-a866b30ea6dc jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. - - {% if shuffled_order == 0 %} - - Sentence A: {{ sentence_good }} - - Sentence B: {{ sentence_bad }} - - {% else %} - - Sentence A: {{ sentence_bad }} - - Sentence B: {{ sentence_good }} - - {% endif %} - - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + {% set shuffled_order_options = [0, 1] | random %} - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" - {% endif %}' + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: better_B_A + original_task: false + name: single_sentence_good_yes_no reference: '' diff --git a/promptsource/templates/blimp/principle_A_domain_3/templates.yaml b/promptsource/templates/blimp/principle_A_domain_3/templates.yaml index cb964877f..da38bba26 100644 --- a/promptsource/templates/blimp/principle_A_domain_3/templates.yaml +++ b/promptsource/templates/blimp/principle_A_domain_3/templates.yaml @@ -1,30 +1,31 @@ dataset: blimp subset: principle_A_domain_3 templates: - 03fe1aff-95fe-49a1-ab6e-5c4c16b22181: !Template + 062eb369-a1b9-4ae5-8f89-f500ccddb0a8: !Template answer_choices: A ||| B - id: 03fe1aff-95fe-49a1-ab6e-5c4c16b22181 + id: 062eb369-a1b9-4ae5-8f89-f500ccddb0a8 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -38,26 +39,29 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: better_A_B reference: '' - 13a812d9-c450-40bd-9a25-481208c79b90: !Template - answer_choices: A ||| B - id: 13a812d9-c450-40bd-9a25-481208c79b90 + 54dbeaa7-cfcf-40d3-b2a8-a022b79761e4: !Template + answer_choices: 1 ||| 2 + id: 54dbeaa7-cfcf-40d3-b2a8-a022b79761e4 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -77,58 +81,29 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B - reference: '' - 146f4806-61a9-43ff-8ac4-ff7f9f5dcc2c: !Template - answer_choices: Yes ||| No - id: 146f4806-61a9-43ff-8ac4-ff7f9f5dcc2c - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes - reference: '' - 25f4d436-274a-4b53-a2d7-63b80bec29c4: !Template - answer_choices: Yes ||| No - id: 25f4d436-274a-4b53-a2d7-63b80bec29c4 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no + name: grammatical_between_1_2 reference: '' - 3f900e5b-df69-459d-b305-364aa5442c77: !Template - answer_choices: 1 ||| 2 - id: 3f900e5b-df69-459d-b305-364aa5442c77 + 580024a2-37e4-4b2b-b8a3-42ce82e95fe8: !Template + answer_choices: A ||| B + id: 580024a2-37e4-4b2b-b8a3-42ce82e95fe8 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -148,17 +123,32 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: grammatical_between_A_B reference: '' - 417732f2-c7e3-4984-84f5-4f6c97027b67: !Template + 72a0475b-45ca-4a3f-a2a9-89be9ba1a1dd: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 72a0475b-45ca-4a3f-a2a9-89be9ba1a1dd + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + a4f146f4-99fb-4461-9d0c-fa53a29d5dae: !Template answer_choices: Yes ||| No - id: 417732f2-c7e3-4984-84f5-4f6c97027b67 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + id: a4f146f4-99fb-4461-9d0c-fa53a29d5dae + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: @@ -166,29 +156,33 @@ templates: original_task: false name: single_sentence_good_yes_no reference: '' - 57f604f4-f765-4362-908a-251b0c5fa157: !Template + b9c16798-57fe-4958-82f4-d7d3e508909c: !Template answer_choices: A ||| B - id: 57f604f4-f765-4362-908a-251b0c5fa157 + id: b9c16798-57fe-4958-82f4-d7d3e508909c jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -202,70 +196,55 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: grammatical_which_one_A_B reference: '' - 57fb6288-488c-42af-b259-681edc6222a7: !Template - answer_choices: 1 ||| 2 - id: 57fb6288-488c-42af-b259-681edc6222a7 + eb65f770-e750-4948-879f-654edc782d5f: !Template + answer_choices: Yes ||| No + id: eb65f770-e750-4948-879f-654edc782d5f jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. - - {% if shuffled_order == 0 %} - - 1: {{ sentence_good }} - - 2: {{ sentence_bad }} + {% set shuffled_order_options = [0, 1] | random %} - {% else %} - - 1: {{ sentence_bad }} - - 2: {{ sentence_good }} + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} - - {% endif %}' + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: grammatical_which_one_1_2 + original_task: false + name: single_sentence_bad_yes_no reference: '' - 683faa0d-31fc-465f-ad84-da5a62b83a83: !Template - answer_choices: A ||| B - id: 683faa0d-31fc-465f-ad84-da5a62b83a83 + f00d14a7-110c-4ee6-98e8-24437696bff2: !Template + answer_choices: 1 ||| 2 + id: f00d14a7-110c-4ee6-98e8-24437696bff2 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -279,43 +258,5 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A - reference: '' - 77c5d27f-4438-4fd2-a1a8-896da83de7d0: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 77c5d27f-4438-4fd2-a1a8-896da83de7d0 - jinja: ' ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: true_null_prompt - reference: '' - 846c2936-6e97-43a9-9e4a-1fe40387bffb: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 846c2936-6e97-43a9-9e4a-1fe40387bffb - jinja: '" ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: null_prompt - reference: '' - a56dac22-38e6-4752-873a-51d024298c21: !Template - answer_choices: Yes ||| No - id: a56dac22-38e6-4752-873a-51d024298c21 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes + name: grammatical_which_one_1_2 reference: '' diff --git a/promptsource/templates/blimp/principle_A_reconstruction/templates.yaml b/promptsource/templates/blimp/principle_A_reconstruction/templates.yaml index 80382b3bd..5951822d8 100644 --- a/promptsource/templates/blimp/principle_A_reconstruction/templates.yaml +++ b/promptsource/templates/blimp/principle_A_reconstruction/templates.yaml @@ -1,66 +1,53 @@ dataset: blimp subset: principle_A_reconstruction templates: - 00394236-354b-4697-b870-a3acb0232856: !Template + 4cf1935d-3ed0-4076-a2fa-d5275392906d: !Template answer_choices: Yes ||| No - id: 00394236-354b-4697-b870-a3acb0232856 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + id: 4cf1935d-3ed0-4076-a2fa-d5275392906d + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_bad_yes_no + name: single_sentence_good_yes_no reference: '' - 15ca5428-e8ab-4c09-93d4-29fb6dfccc65: !Template - answer_choices: A ||| B - id: 15ca5428-e8ab-4c09-93d4-29fb6dfccc65 + 7a7e470f-87ba-4f93-bffc-388328701910: !Template + answer_choices: Yes ||| No + id: 7a7e470f-87ba-4f93-bffc-388328701910 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. - - {% if shuffled_order == 0 %} - - A: {{ sentence_good }} - - B: {{ sentence_bad }} - - {% else %} - - A: {{ sentence_bad }} - - B: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - {{ answer_choices[0] }} - - {% else %} + {% set shuffled_order_options = [0, 1] | random %} - {{ answer_choices[1] }} + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" - {% endif %}' + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: grammatical_between_A_B + original_task: false + name: single_sentence_bad_yes_no reference: '' - 1d7f7749-e21b-42ae-a7f2-03a378f1dd77: !Template + 883a8e51-9890-4ef6-942d-255b47a7d724: !Template answer_choices: A ||| B - id: 1d7f7749-e21b-42ae-a7f2-03a378f1dd77 + id: 883a8e51-9890-4ef6-942d-255b47a7d724 jinja: '{% set shuffled_order = [0, 1] | random %} + {% set shuffled_order_options = [0, 1] | random %} + Read the following two sentences. {% if shuffled_order == 0 %} @@ -77,8 +64,8 @@ templates: {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -94,12 +81,15 @@ templates: original_task: true name: better_A_B reference: '' - 427171c8-e1c9-4c97-a55d-ab7757bdd0c8: !Template + 99fbf5d8-c746-4eb9-81ae-21dec2382ef5: !Template answer_choices: 1 ||| 2 - id: 427171c8-e1c9-4c97-a55d-ab7757bdd0c8 + id: 99fbf5d8-c746-4eb9-81ae-21dec2382ef5 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} @@ -131,31 +121,35 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: grammatical_between_1_2 reference: '' - 4bd7341a-5539-4afb-a791-92744dc256d5: !Template - answer_choices: A ||| B - id: 4bd7341a-5539-4afb-a791-92744dc256d5 + abc3a6ea-baac-4f8f-9a7e-f3e4242ecc46: !Template + answer_choices: 1 ||| 2 + id: abc3a6ea-baac-4f8f-9a7e-f3e4242ecc46 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -169,42 +163,29 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A - reference: '' - 56342922-df71-4cfe-8424-560b1c2a7a24: !Template - answer_choices: Yes ||| No - id: 56342922-df71-4cfe-8424-560b1c2a7a24 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes + name: grammatical_which_one_1_2 reference: '' - 6ae9a244-bfaa-443b-a0d0-34856b47fd68: !Template - answer_choices: 1 ||| 2 - id: 6ae9a244-bfaa-443b-a0d0-34856b47fd68 + bc16c071-ea83-488e-bc4c-fecd07048451: !Template + answer_choices: A ||| B + id: bc16c071-ea83-488e-bc4c-fecd07048451 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -224,25 +205,17 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 - reference: '' - 96921ce7-cc0b-40ea-ae48-d28ead268e8c: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 96921ce7-cc0b-40ea-ae48-d28ead268e8c - jinja: '" ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: null_prompt + name: grammatical_between_A_B reference: '' - 9a3b8431-b8fc-49a3-9dd5-5de7c8f516f7: !Template + e3a2610f-0d79-4f09-a958-05df12dba4e6: !Template answer_choices: A ||| B - id: 9a3b8431-b8fc-49a3-9dd5-5de7c8f516f7 + id: e3a2610f-0d79-4f09-a958-05df12dba4e6 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} @@ -276,46 +249,14 @@ templates: original_task: true name: grammatical_which_one_A_B reference: '' - c0213958-4f02-40c4-8b9e-9ba412d609dd: !Template - answer_choices: Yes ||| No - id: c0213958-4f02-40c4-8b9e-9ba412d609dd - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no - reference: '' - c631546b-227c-4a99-a94f-f8420c4a1312: !Template + ec07a3c1-fbb3-46eb-ab9a-c891941ea8d5: !Template answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: c631546b-227c-4a99-a94f-f8420c4a1312 + id: ec07a3c1-fbb3-46eb-ab9a-c891941ea8d5 jinja: ' ||| {{ sentence_good }}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - Accuracy original_task: true - name: true_null_prompt - reference: '' - ef661aa7-70a7-472b-b9a8-7e8188274704: !Template - answer_choices: Yes ||| No - id: ef661aa7-70a7-472b-b9a8-7e8188274704 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes + name: null_prompt reference: '' diff --git a/promptsource/templates/blimp/regular_plural_subject_verb_agreement_1/templates.yaml b/promptsource/templates/blimp/regular_plural_subject_verb_agreement_1/templates.yaml index 45a94b975..a080404bc 100644 --- a/promptsource/templates/blimp/regular_plural_subject_verb_agreement_1/templates.yaml +++ b/promptsource/templates/blimp/regular_plural_subject_verb_agreement_1/templates.yaml @@ -1,40 +1,33 @@ dataset: blimp subset: regular_plural_subject_verb_agreement_1 templates: - 1014056b-4114-4213-85bf-9718961144ad: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 1014056b-4114-4213-85bf-9718961144ad - jinja: '" ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: null_prompt - reference: '' - 11a581f6-aa83-4235-8840-4d8348b9f9ec: !Template + 3cb9bff1-d20f-4370-b12e-aa0aba3756b9: !Template answer_choices: A ||| B - id: 11a581f6-aa83-4235-8840-4d8348b9f9ec + id: 3cb9bff1-d20f-4370-b12e-aa0aba3756b9 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -48,31 +41,46 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: grammatical_between_A_B reference: '' - 11f69f2f-e468-47f5-8910-13c9e463b357: !Template - answer_choices: A ||| B - id: 11f69f2f-e468-47f5-8910-13c9e463b357 + 43a65a43-4d6c-4aaa-ae1f-6ea76aa09511: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 43a65a43-4d6c-4aaa-ae1f-6ea76aa09511 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + 584848e5-8a5e-42fc-bf70-433eec10c8f8: !Template + answer_choices: 1 ||| 2 + id: 584848e5-8a5e-42fc-bf70-433eec10c8f8 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -86,14 +94,17 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: grammatical_between_1_2 reference: '' - 30935e75-402b-432f-b81a-02841ae03398: !Template + 5cc330d7-d3a9-4fd9-bdea-cdee4c31401c: !Template answer_choices: 1 ||| 2 - id: 30935e75-402b-432f-b81a-02841ae03398 + id: 5cc330d7-d3a9-4fd9-bdea-cdee4c31401c jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} @@ -127,46 +138,31 @@ templates: original_task: true name: grammatical_which_one_1_2 reference: '' - 686309dd-f772-462a-be99-e6c3af95ddd2: !Template - answer_choices: Yes ||| No - id: 686309dd-f772-462a-be99-e6c3af95ddd2 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes - reference: '' - 8fa5dae0-cd60-4e35-9beb-ac5e5b49e513: !Template + 5e152d33-d1ff-41a0-a762-e2cdc7148ace: !Template answer_choices: A ||| B - id: 8fa5dae0-cd60-4e35-9beb-ac5e5b49e513 + id: 5e152d33-d1ff-41a0-a762-e2cdc7148ace jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -180,85 +176,49 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: better_A_B reference: '' - 93384294-5451-41de-8c2f-4b2937d16d14: !Template + 9da7bddc-fd8b-4fff-a068-5af8d867f529: !Template answer_choices: Yes ||| No - id: 93384294-5451-41de-8c2f-4b2937d16d14 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + id: 9da7bddc-fd8b-4fff-a068-5af8d867f529 + jinja: '{% set shuffled_order = [0, 1] | random %} - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes - reference: '' - 98a04913-0d5e-4e10-96b4-b8529d34c4ee: !Template - answer_choices: Yes ||| No - id: 98a04913-0d5e-4e10-96b4-b8529d34c4ee - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + {% set shuffled_order_options = [0, 1] | random %} - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no - reference: '' - b7f37976-d243-4fda-98ad-3024e56fd3e9: !Template - answer_choices: Yes ||| No - id: b7f37976-d243-4fda-98ad-3024e56fd3e9 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_good_yes_no - reference: '' - e95f907b-9301-46c3-a24b-90f675831f5e: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: e95f907b-9301-46c3-a24b-90f675831f5e - jinja: ' ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: true_null_prompt + name: single_sentence_bad_yes_no reference: '' - eb950807-9957-4685-a205-8f9280cb6961: !Template - answer_choices: 1 ||| 2 - id: eb950807-9957-4685-a205-8f9280cb6961 + a63d58b4-8dab-4f53-aa13-efa191bc0d0c: !Template + answer_choices: A ||| B + id: a63d58b4-8dab-4f53-aa13-efa191bc0d0c jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -278,44 +238,25 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: grammatical_which_one_A_B reference: '' - fed0ceb0-61d5-4e9f-af5d-8268de202a15: !Template - answer_choices: A ||| B - id: fed0ceb0-61d5-4e9f-af5d-8268de202a15 + e83f2ca0-0c6a-4cf6-88ca-4533c8297618: !Template + answer_choices: Yes ||| No + id: e83f2ca0-0c6a-4cf6-88ca-4533c8297618 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. - - {% if shuffled_order == 0 %} - - A: {{ sentence_good }} + {% set shuffled_order_options = [0, 1] | random %} - B: {{ sentence_bad }} - - {% else %} - - A: {{ sentence_bad }} - - B: {{ sentence_good }} + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} - - {% endif %}' + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: grammatical_which_one_A_B + original_task: false + name: single_sentence_good_yes_no reference: '' diff --git a/promptsource/templates/blimp/regular_plural_subject_verb_agreement_2/templates.yaml b/promptsource/templates/blimp/regular_plural_subject_verb_agreement_2/templates.yaml index a2a1d6552..2fde9c0ec 100644 --- a/promptsource/templates/blimp/regular_plural_subject_verb_agreement_2/templates.yaml +++ b/promptsource/templates/blimp/regular_plural_subject_verb_agreement_2/templates.yaml @@ -1,29 +1,33 @@ dataset: blimp subset: regular_plural_subject_verb_agreement_2 templates: - 1a3280a2-fc7d-4e83-9f9f-7e2f296ba77a: !Template - answer_choices: A ||| B - id: 1a3280a2-fc7d-4e83-9f9f-7e2f296ba77a + 3e972e06-8b0c-4a5b-8974-9ddff6808924: !Template + answer_choices: 1 ||| 2 + id: 3e972e06-8b0c-4a5b-8974-9ddff6808924 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -37,14 +41,28 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: grammatical_between_1_2 reference: '' - 2d45742a-6652-493f-abab-cd92bc638151: !Template + 6e2d13c8-ad7b-409d-b32d-e8ae75ea7efa: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 6e2d13c8-ad7b-409d-b32d-e8ae75ea7efa + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + 7663aff0-04d8-4005-ba53-9b29f6bf0459: !Template answer_choices: A ||| B - id: 2d45742a-6652-493f-abab-cd92bc638151 + id: 7663aff0-04d8-4005-ba53-9b29f6bf0459 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} @@ -78,45 +96,33 @@ templates: original_task: true name: grammatical_which_one_A_B reference: '' - 3346144c-78a2-41e9-97b6-3150c25f55bc: !Template - answer_choices: Yes ||| No - id: 3346144c-78a2-41e9-97b6-3150c25f55bc - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no - reference: '' - 69538248-50fb-4b7f-bb30-0902152a9a33: !Template - answer_choices: A ||| B - id: 69538248-50fb-4b7f-bb30-0902152a9a33 + 7f72e931-7cc8-459c-8a72-f72da1f38165: !Template + answer_choices: 1 ||| 2 + id: 7f72e931-7cc8-459c-8a72-f72da1f38165 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -130,102 +136,33 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A - reference: '' - 6cdcf461-ef6c-4a1d-a039-3709a6730f7d: !Template - answer_choices: Yes ||| No - id: 6cdcf461-ef6c-4a1d-a039-3709a6730f7d - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no - reference: '' - 6f178a7e-43b4-4ebe-98fe-58a4dc61e430: !Template - answer_choices: Yes ||| No - id: 6f178a7e-43b4-4ebe-98fe-58a4dc61e430 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes - reference: '' - 6fc23973-257b-445f-9ca6-b341240eaae5: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 6fc23973-257b-445f-9ca6-b341240eaae5 - jinja: '" ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: null_prompt - reference: '' - 7b829592-2c44-45c2-a092-1165f9012e1a: !Template - answer_choices: Yes ||| No - id: 7b829592-2c44-45c2-a092-1165f9012e1a - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes - reference: '' - 9fbf1297-64a5-403a-bb3b-2799a0cae9fc: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 9fbf1297-64a5-403a-bb3b-2799a0cae9fc - jinja: ' ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: true_null_prompt + name: grammatical_which_one_1_2 reference: '' - d14843af-6477-451e-814f-c0e50112732b: !Template - answer_choices: 1 ||| 2 - id: d14843af-6477-451e-814f-c0e50112732b + 95d63d7e-0bb8-4d18-bd4e-b727a5ae1c8f: !Template + answer_choices: A ||| B + id: 95d63d7e-0bb8-4d18-bd4e-b727a5ae1c8f jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -239,14 +176,37 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: better_A_B + reference: '' + ad7f9dd2-67ed-46cc-ad1e-f8c03bb2854d: !Template + answer_choices: Yes ||| No + id: ad7f9dd2-67ed-46cc-ad1e-f8c03bb2854d + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no reference: '' - d2c61505-b860-4812-bb63-ebcb015a226b: !Template + c8735e72-1665-442d-9170-78c2fbf308e7: !Template answer_choices: A ||| B - id: d2c61505-b860-4812-bb63-ebcb015a226b + id: c8735e72-1665-442d-9170-78c2fbf308e7 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} @@ -280,42 +240,23 @@ templates: original_task: true name: grammatical_between_A_B reference: '' - d9523177-55ba-4053-9504-70421382bec8: !Template - answer_choices: 1 ||| 2 - id: d9523177-55ba-4053-9504-70421382bec8 + f9d3b6b2-93ff-41d8-99c0-140462a8da47: !Template + answer_choices: Yes ||| No + id: f9d3b6b2-93ff-41d8-99c0-140462a8da47 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. - - {% if shuffled_order == 0 %} - - 1: {{ sentence_good }} - - 2: {{ sentence_bad }} - - {% else %} - - 1: {{ sentence_bad }} - - 2: {{ sentence_good }} - - {% endif %} + {% set shuffled_order_options = [0, 1] | random %} - ||| - - {% if shuffled_order == 0 %} - - {{ answer_choices[0] }} + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" - {% else %} - - {{ answer_choices[1] }} - - {% endif %}' + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: grammatical_between_1_2 + original_task: false + name: single_sentence_good_yes_no reference: '' diff --git a/promptsource/templates/blimp/sentential_negation_npi_licensor_present/templates.yaml b/promptsource/templates/blimp/sentential_negation_npi_licensor_present/templates.yaml index 34f989864..1248c63a2 100644 --- a/promptsource/templates/blimp/sentential_negation_npi_licensor_present/templates.yaml +++ b/promptsource/templates/blimp/sentential_negation_npi_licensor_present/templates.yaml @@ -1,56 +1,27 @@ dataset: blimp subset: sentential_negation_npi_licensor_present templates: - 036fa9f4-c62a-4bf6-a7fa-e27c1e632653: !Template - answer_choices: Yes ||| No - id: 036fa9f4-c62a-4bf6-a7fa-e27c1e632653 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no - reference: '' - 0516d2b3-9634-40b7-a972-91be2d548991: !Template - answer_choices: Yes ||| No - id: 0516d2b3-9634-40b7-a972-91be2d548991 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes - reference: '' - 14a1e0d8-4a24-496d-a559-1d2564cf9286: !Template - answer_choices: 1 ||| 2 - id: 14a1e0d8-4a24-496d-a559-1d2564cf9286 + 20ea7037-f4bc-4d75-b51c-e76b702ab2af: !Template + answer_choices: A ||| B + id: 20ea7037-f4bc-4d75-b51c-e76b702ab2af jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -70,70 +41,55 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: grammatical_between_A_B reference: '' - 333be7e9-ec2d-45b0-87c3-06506dcbc25a: !Template - answer_choices: 1 ||| 2 - id: 333be7e9-ec2d-45b0-87c3-06506dcbc25a + 3429788f-02cc-45ef-b33f-9c787470c350: !Template + answer_choices: Yes ||| No + id: 3429788f-02cc-45ef-b33f-9c787470c350 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. - - {% if shuffled_order == 0 %} - - 1: {{ sentence_good }} + {% set shuffled_order_options = [0, 1] | random %} - 2: {{ sentence_bad }} + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" - {% else %} - - 1: {{ sentence_bad }} - - 2: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} - - {% endif %}' + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: grammatical_which_one_1_2 + original_task: false + name: single_sentence_good_yes_no reference: '' - 791a5874-dfc3-4ddf-be8c-7db76caa9a53: !Template + 3589d43a-91b0-4e94-96d7-0b4ea7f2ae26: !Template answer_choices: A ||| B - id: 791a5874-dfc3-4ddf-be8c-7db76caa9a53 + id: 3589d43a-91b0-4e94-96d7-0b4ea7f2ae26 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -147,58 +103,55 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: grammatical_which_one_A_B reference: '' - b6240669-85c7-4129-aae3-5327cd0a8b3c: !Template + 4a8cffd4-cfb2-45d4-af72-e4b47a8fa8b7: !Template answer_choices: Yes ||| No - id: b6240669-85c7-4129-aae3-5327cd0a8b3c - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + id: 4a8cffd4-cfb2-45d4-af72-e4b47a8fa8b7 + jinja: '{% set shuffled_order = [0, 1] | random %} - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_bad_no_yes - reference: '' - b90b77dc-ed80-42ce-97d6-6fbd08741c4c: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: b90b77dc-ed80-42ce-97d6-6fbd08741c4c - jinja: ' ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: true_null_prompt + name: single_sentence_bad_yes_no reference: '' - ca2e8271-bf69-4575-8ccd-406d8cf3140c: !Template - answer_choices: A ||| B - id: ca2e8271-bf69-4575-8ccd-406d8cf3140c + 67c9bf91-abfe-4d38-bab6-2b0f8f16833e: !Template + answer_choices: 1 ||| 2 + id: 67c9bf91-abfe-4d38-bab6-2b0f8f16833e jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -212,32 +165,33 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: grammatical_which_one_1_2 reference: '' - df82eee5-5fb0-4ec3-ad76-22483ca78977: !Template + 7091f7b6-b026-4f2d-a4d2-11da09bdb544: !Template answer_choices: A ||| B - id: df82eee5-5fb0-4ec3-ad76-22483ca78977 + id: 7091f7b6-b026-4f2d-a4d2-11da09bdb544 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -251,26 +205,40 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: better_A_B reference: '' - e10ae3f2-4bee-44fd-9062-0f66b6093055: !Template - answer_choices: A ||| B - id: e10ae3f2-4bee-44fd-9062-0f66b6093055 + 76c8e7a2-3746-4276-9b1e-ec34bf338bbc: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 76c8e7a2-3746-4276-9b1e-ec34bf338bbc + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + 9cd5bc3a-f7a7-4854-a32c-05f552b1e59b: !Template + answer_choices: 1 ||| 2 + id: 9cd5bc3a-f7a7-4854-a32c-05f552b1e59b jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -290,32 +258,5 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B - reference: '' - e23d2087-8e37-42ed-a2f8-b4fb98f9277c: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: e23d2087-8e37-42ed-a2f8-b4fb98f9277c - jinja: '" ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: null_prompt - reference: '' - f138662a-f1d4-49f7-9b61-96af87329a16: !Template - answer_choices: Yes ||| No - id: f138662a-f1d4-49f7-9b61-96af87329a16 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no + name: grammatical_between_1_2 reference: '' diff --git a/promptsource/templates/blimp/sentential_negation_npi_scope/templates.yaml b/promptsource/templates/blimp/sentential_negation_npi_scope/templates.yaml index ce045818d..6c787077e 100644 --- a/promptsource/templates/blimp/sentential_negation_npi_scope/templates.yaml +++ b/promptsource/templates/blimp/sentential_negation_npi_scope/templates.yaml @@ -1,66 +1,15 @@ dataset: blimp subset: sentential_negation_npi_scope templates: - 21a9ee61-c118-48d8-8230-9562e93b46ab: !Template - answer_choices: Yes ||| No - id: 21a9ee61-c118-48d8-8230-9562e93b46ab - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no - reference: '' - 330c84ad-8cb3-43a7-ac74-9b7fbcf47f0e: !Template - answer_choices: A ||| B - id: 330c84ad-8cb3-43a7-ac74-9b7fbcf47f0e - jinja: '{% set shuffled_order = [0, 1] | random %} - - Read the following two sentences. - - {% if shuffled_order == 0 %} - - Sentence A: {{ sentence_good }} - - Sentence B: {{ sentence_bad }} - - {% else %} - - Sentence A: {{ sentence_bad }} - - Sentence B: {{ sentence_good }} - - {% endif %} - - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} - - {% endif %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: better_B_A - reference: '' - 4e94ed9f-bff1-4859-8b8f-fd6901325b79: !Template + 06bcd940-bd0b-469d-8270-5c10ce6724d6: !Template answer_choices: 1 ||| 2 - id: 4e94ed9f-bff1-4859-8b8f-fd6901325b79 + id: 06bcd940-bd0b-469d-8270-5c10ce6724d6 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} @@ -94,12 +43,15 @@ templates: original_task: true name: grammatical_which_one_1_2 reference: '' - 568cc2c8-2d7a-4988-9cc4-65641f742a86: !Template + 24be7bee-d104-47bd-9dd0-dd6feaa5a50c: !Template answer_choices: A ||| B - id: 568cc2c8-2d7a-4988-9cc4-65641f742a86 + id: 24be7bee-d104-47bd-9dd0-dd6feaa5a50c jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} @@ -133,30 +85,51 @@ templates: original_task: true name: grammatical_between_A_B reference: '' - 5c224b82-eca5-44c0-b9db-9730dcc69238: !Template + 29748319-4f33-48b9-94a3-b149623688af: !Template + answer_choices: Yes ||| No + id: 29748319-4f33-48b9-94a3-b149623688af + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 8077910e-4d3f-440f-aec8-cb8d5026add7: !Template answer_choices: A ||| B - id: 5c224b82-eca5-44c0-b9db-9730dcc69238 + id: 8077910e-4d3f-440f-aec8-cb8d5026add7 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -170,12 +143,12 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: better_A_B reference: '' - 657e2c7d-fe98-4370-83f9-a766eae08819: !Template + ceaf4bf2-6a63-4d0f-88e5-da569756bfdc: !Template answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 657e2c7d-fe98-4370-83f9-a766eae08819 - jinja: '" ||| {{ sentence_good }}' + id: ceaf4bf2-6a63-4d0f-88e5-da569756bfdc + jinja: ' ||| {{ sentence_good }}' metadata: !TemplateMetadata choices_in_prompt: false metrics: @@ -183,61 +156,53 @@ templates: original_task: true name: null_prompt reference: '' - 9886e05d-9bf1-40b2-b977-55edd03132ea: !Template + d0b6a823-b8a8-4fc0-931c-4fdd59aa754f: !Template answer_choices: Yes ||| No - id: 9886e05d-9bf1-40b2-b977-55edd03132ea - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + id: d0b6a823-b8a8-4fc0-931c-4fdd59aa754f + jinja: '{% set shuffled_order = [0, 1] | random %} - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes - reference: '' - c70ee875-e604-4166-a5e6-1afb4516dfbb: !Template - answer_choices: Yes ||| No - id: c70ee875-e604-4166-a5e6-1afb4516dfbb - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + {% set shuffled_order_options = [0, 1] | random %} - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_good_no_yes + name: single_sentence_bad_yes_no reference: '' - cf464250-783d-4251-b039-b4dd4d93e9c0: !Template + d8a3adbb-8a01-424a-a20d-334b5649c20d: !Template answer_choices: A ||| B - id: cf464250-783d-4251-b039-b4dd4d93e9c0 + id: d8a3adbb-8a01-424a-a20d-334b5649c20d jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -251,25 +216,17 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B - reference: '' - d5d31a73-10d0-437d-a80e-60689045246e: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: d5d31a73-10d0-437d-a80e-60689045246e - jinja: ' ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: true_null_prompt + name: grammatical_which_one_A_B reference: '' - eb95abfa-ed88-49f0-aaaa-518ec91f069e: !Template + e1b2a2c6-c9e8-4836-ab95-7607af2f0baa: !Template answer_choices: 1 ||| 2 - id: eb95abfa-ed88-49f0-aaaa-518ec91f069e + id: e1b2a2c6-c9e8-4836-ab95-7607af2f0baa jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} @@ -303,19 +260,3 @@ templates: original_task: true name: grammatical_between_1_2 reference: '' - ed344823-e633-4cfb-a81b-20f7991647a5: !Template - answer_choices: Yes ||| No - id: ed344823-e633-4cfb-a81b-20f7991647a5 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no - reference: '' diff --git a/promptsource/templates/blimp/sentential_subject_island/templates.yaml b/promptsource/templates/blimp/sentential_subject_island/templates.yaml index 5fb6caf02..150e21055 100644 --- a/promptsource/templates/blimp/sentential_subject_island/templates.yaml +++ b/promptsource/templates/blimp/sentential_subject_island/templates.yaml @@ -1,10 +1,10 @@ dataset: blimp subset: sentential_subject_island templates: - 09965253-1a19-47d0-9da0-1aa906090e23: !Template + 33c5eb89-7d3f-4c06-b891-f76ebb029fa2: !Template answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 09965253-1a19-47d0-9da0-1aa906090e23 - jinja: '" ||| {{ sentence_good }}' + id: 33c5eb89-7d3f-4c06-b891-f76ebb029fa2 + jinja: ' ||| {{ sentence_good }}' metadata: !TemplateMetadata choices_in_prompt: false metrics: @@ -12,12 +12,55 @@ templates: original_task: true name: null_prompt reference: '' - 26f49f2a-8c9e-44d9-9f88-cd5cbdee4c1b: !Template + 3bcabbab-d834-400b-8004-47c7af8d0296: !Template + answer_choices: Yes ||| No + id: 3bcabbab-d834-400b-8004-47c7af8d0296 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 5f943d95-000b-4d16-9de9-b41221af3f2a: !Template + answer_choices: Yes ||| No + id: 5f943d95-000b-4d16-9de9-b41221af3f2a + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 63fb3dc7-4361-4b92-806a-003bb95d3cbc: !Template answer_choices: 1 ||| 2 - id: 26f49f2a-8c9e-44d9-9f88-cd5cbdee4c1b + id: 63fb3dc7-4361-4b92-806a-003bb95d3cbc jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} @@ -51,30 +94,31 @@ templates: original_task: true name: grammatical_which_one_1_2 reference: '' - 3e615c5b-16f2-46fd-aafd-3c951c6a8b6f: !Template + 8f8e59e9-9f15-45cc-bb1b-bb36f0b514bd: !Template answer_choices: A ||| B - id: 3e615c5b-16f2-46fd-aafd-3c951c6a8b6f + id: 8f8e59e9-9f15-45cc-bb1b-bb36f0b514bd jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -88,30 +132,17 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B - reference: '' - 5fe40b27-b394-4048-9f8c-e52e19bf8531: !Template - answer_choices: Yes ||| No - id: 5fe40b27-b394-4048-9f8c-e52e19bf8531 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no + name: better_A_B reference: '' - 6c18d59c-346a-4244-90da-97cf5183c3c2: !Template + e7d92196-f32b-4922-a2b0-9d70977e1708: !Template answer_choices: 1 ||| 2 - id: 6c18d59c-346a-4244-90da-97cf5183c3c2 + id: e7d92196-f32b-4922-a2b0-9d70977e1708 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} @@ -145,84 +176,34 @@ templates: original_task: true name: grammatical_between_1_2 reference: '' - 6c7f8921-fd6d-424a-bad8-0793619eb265: !Template - answer_choices: Yes ||| No - id: 6c7f8921-fd6d-424a-bad8-0793619eb265 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes - reference: '' - 71eb6673-8bc7-421d-9825-1a05270b90d3: !Template + f8a5abf1-e833-4095-b1c6-ed0f69f8353b: !Template answer_choices: A ||| B - id: 71eb6673-8bc7-421d-9825-1a05270b90d3 + id: f8a5abf1-e833-4095-b1c6-ed0f69f8353b jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} - - {% endif %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: better_B_A - reference: '' - 7f841f3f-5a05-47d2-b1c3-2ffd297eaf09: !Template - answer_choices: A ||| B - id: 7f841f3f-5a05-47d2-b1c3-2ffd297eaf09 - jinja: '{% set shuffled_order = [0, 1] | random %} - - Read the following two sentences. + ||| {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} - - Sentence B: {{ sentence_bad }} - - {% else %} - - Sentence A: {{ sentence_bad }} - - Sentence B: {{ sentence_good }} - - {% endif %} - - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} - {{ answer_choices[0] }} {% else %} @@ -235,57 +216,17 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B - reference: '' - 7fe9d8f2-728d-41eb-ada2-32919cb9ca86: !Template - answer_choices: Yes ||| No - id: 7fe9d8f2-728d-41eb-ada2-32919cb9ca86 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes - reference: '' - c17ecf33-f485-4c0a-86a4-c672a78f29e8: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: c17ecf33-f485-4c0a-86a4-c672a78f29e8 - jinja: ' ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: true_null_prompt - reference: '' - c7f6ee4a-2340-4082-96c4-df9cd51dc3de: !Template - answer_choices: Yes ||| No - id: c7f6ee4a-2340-4082-96c4-df9cd51dc3de - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no + name: grammatical_between_A_B reference: '' - e37d7e7a-bf02-4975-8e0b-a2587b82a62c: !Template + f98877e8-23c3-4c3a-9747-c96b60444de2: !Template answer_choices: A ||| B - id: e37d7e7a-bf02-4975-8e0b-a2587b82a62c + id: f98877e8-23c3-4c3a-9747-c96b60444de2 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} @@ -317,5 +258,5 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: grammatical_which_one_A_B reference: '' diff --git a/promptsource/templates/blimp/superlative_quantifiers_1/templates.yaml b/promptsource/templates/blimp/superlative_quantifiers_1/templates.yaml index 22786a5dc..6a95b51d7 100644 --- a/promptsource/templates/blimp/superlative_quantifiers_1/templates.yaml +++ b/promptsource/templates/blimp/superlative_quantifiers_1/templates.yaml @@ -1,12 +1,15 @@ dataset: blimp subset: superlative_quantifiers_1 templates: - 04d7705a-1887-4442-b1e5-c98417693344: !Template + 09ffbdea-f522-48ff-854a-c2b0a8864e46: !Template answer_choices: 1 ||| 2 - id: 04d7705a-1887-4442-b1e5-c98417693344 + id: 09ffbdea-f522-48ff-854a-c2b0a8864e46 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} @@ -40,12 +43,15 @@ templates: original_task: true name: grammatical_between_1_2 reference: '' - 07c9edae-c9df-4d4e-ad29-cbf68f2748b5: !Template + 2b5f03af-6ae6-4c97-bdf9-25bfde671223: !Template answer_choices: A ||| B - id: 07c9edae-c9df-4d4e-ad29-cbf68f2748b5 + id: 2b5f03af-6ae6-4c97-bdf9-25bfde671223 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} @@ -77,30 +83,17 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B - reference: '' - 0befc6d8-ac12-4a6f-bc96-0a19fece81d9: !Template - answer_choices: Yes ||| No - id: 0befc6d8-ac12-4a6f-bc96-0a19fece81d9 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no + name: grammatical_which_one_A_B reference: '' - 0f5d590d-1668-46a4-a080-a97d148545f0: !Template + ac839ac0-5de1-44d8-8d8b-a0a074e4baac: !Template answer_choices: A ||| B - id: 0f5d590d-1668-46a4-a080-a97d148545f0 + id: ac839ac0-5de1-44d8-8d8b-a0a074e4baac jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} @@ -132,73 +125,17 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B - reference: '' - 123d4fa9-8527-4c8b-8418-ded4443a1575: !Template - answer_choices: Yes ||| No - id: 123d4fa9-8527-4c8b-8418-ded4443a1575 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes - reference: '' - 1c438fa3-398d-48fe-b41c-4be44bf197cb: !Template - answer_choices: Yes ||| No - id: 1c438fa3-398d-48fe-b41c-4be44bf197cb - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes - reference: '' - 3d37f387-e142-4e4e-b741-a3d3535d5df8: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 3d37f387-e142-4e4e-b741-a3d3535d5df8 - jinja: ' ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: true_null_prompt - reference: '' - 452041bb-fcd3-4eb9-96cc-d69465a96c9f: !Template - answer_choices: Yes ||| No - id: 452041bb-fcd3-4eb9-96cc-d69465a96c9f - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no + name: grammatical_between_A_B reference: '' - 6235e720-b936-4ebf-9c3f-08e958e84015: !Template + bd64feeb-3c96-441d-972f-36a6c3a2091d: !Template answer_choices: 1 ||| 2 - id: 6235e720-b936-4ebf-9c3f-08e958e84015 + id: bd64feeb-3c96-441d-972f-36a6c3a2091d jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} @@ -232,49 +169,64 @@ templates: original_task: true name: grammatical_which_one_1_2 reference: '' - 75fe932f-d045-4bd6-815b-ed1b2ca7ebdd: !Template - answer_choices: A ||| B - id: 75fe932f-d045-4bd6-815b-ed1b2ca7ebdd + bdc91436-3ced-43cb-984a-5964900cdde1: !Template + answer_choices: Yes ||| No + id: bdc91436-3ced-43cb-984a-5964900cdde1 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. - - {% if shuffled_order == 0 %} + {% set shuffled_order_options = [0, 1] | random %} - Sentence A: {{ sentence_good }} - - Sentence B: {{ sentence_bad }} - - {% else %} - - Sentence A: {{ sentence_bad }} - - Sentence B: {{ sentence_good }} + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" - {% endif %} - - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} - - {{ answer_choices[0] }} + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + cdbe007d-c8a1-4fa9-9fa0-1d171337c7d6: !Template + answer_choices: Yes ||| No + id: cdbe007d-c8a1-4fa9-9fa0-1d171337c7d6 + jinja: '{% set shuffled_order = [0, 1] | random %} - {% else %} + {% set shuffled_order_options = [0, 1] | random %} - {{ answer_choices[1] }} + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" - {% endif %}' + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + d9516b74-8b12-4042-bec4-b8266c021428: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: d9516b74-8b12-4042-bec4-b8266c021428 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy original_task: true - name: better_A_B + name: null_prompt reference: '' - 7d28c5c3-81fe-4dd9-839d-c2e83966f292: !Template + f69f8b6d-2bfe-4e88-83a5-d38a46ef121c: !Template answer_choices: A ||| B - id: 7d28c5c3-81fe-4dd9-839d-c2e83966f292 + id: f69f8b6d-2bfe-4e88-83a5-d38a46ef121c jinja: '{% set shuffled_order = [0, 1] | random %} + {% set shuffled_order_options = [0, 1] | random %} + Read the following two sentences. {% if shuffled_order == 0 %} @@ -291,8 +243,8 @@ templates: {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -306,16 +258,5 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A - reference: '' - 9abe04b0-aebd-4095-a72a-51d2fa1ae515: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 9abe04b0-aebd-4095-a72a-51d2fa1ae515 - jinja: '" ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: null_prompt + name: better_A_B reference: '' diff --git a/promptsource/templates/blimp/superlative_quantifiers_2/templates.yaml b/promptsource/templates/blimp/superlative_quantifiers_2/templates.yaml index e50c6336a..1eb53b205 100644 --- a/promptsource/templates/blimp/superlative_quantifiers_2/templates.yaml +++ b/promptsource/templates/blimp/superlative_quantifiers_2/templates.yaml @@ -1,39 +1,66 @@ dataset: blimp subset: superlative_quantifiers_2 templates: - 07a61dea-60bb-4846-aea2-520dd0177768: !Template - answer_choices: Yes ||| No - id: 07a61dea-60bb-4846-aea2-520dd0177768 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + 28e41b28-a85d-4caf-b29b-2ff3aa909a42: !Template + answer_choices: A ||| B + id: 28e41b28-a85d-4caf-b29b-2ff3aa909a42 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: false - name: single_sentence_bad_yes_no + original_task: true + name: better_A_B reference: '' - 4390642c-6af5-41b5-a624-3abd21582de1: !Template + 39ef3b76-021e-48cd-83dc-00ff72f5a5e2: !Template answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 4390642c-6af5-41b5-a624-3abd21582de1 + id: 39ef3b76-021e-48cd-83dc-00ff72f5a5e2 jinja: ' ||| {{ sentence_good }}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - Accuracy original_task: true - name: true_null_prompt + name: null_prompt reference: '' - 6161ef6e-2a0b-4f08-84ad-0a91512d588b: !Template + 6bc37d1d-5868-4d84-92a1-0fdfada218d8: !Template answer_choices: 1 ||| 2 - id: 6161ef6e-2a0b-4f08-84ad-0a91512d588b + id: 6bc37d1d-5868-4d84-92a1-0fdfada218d8 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} @@ -67,12 +94,15 @@ templates: original_task: true name: grammatical_between_1_2 reference: '' - 79299f1c-b62e-488f-b403-f026155c7e7b: !Template + 860f5327-3af1-42dc-866f-dc310cf1b90f: !Template answer_choices: A ||| B - id: 79299f1c-b62e-488f-b403-f026155c7e7b + id: 860f5327-3af1-42dc-866f-dc310cf1b90f jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} @@ -104,118 +134,57 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B - reference: '' - 819a1ab8-aa4c-4d7d-a5d2-ddb2c54cdc86: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 819a1ab8-aa4c-4d7d-a5d2-ddb2c54cdc86 - jinja: '" ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: null_prompt + name: grammatical_between_A_B reference: '' - 874c2efc-10a6-4b5c-9034-e4a00dbdeb4a: !Template - answer_choices: A ||| B - id: 874c2efc-10a6-4b5c-9034-e4a00dbdeb4a + b8c6f707-38ef-464d-8df6-7132f159426e: !Template + answer_choices: Yes ||| No + id: b8c6f707-38ef-464d-8df6-7132f159426e jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. - - {% if shuffled_order == 0 %} - - Sentence A: {{ sentence_good }} - - Sentence B: {{ sentence_bad }} - - {% else %} - - Sentence A: {{ sentence_bad }} - - Sentence B: {{ sentence_good }} - - {% endif %} - - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} - - {{ answer_choices[0] }} + {% set shuffled_order_options = [0, 1] | random %} - {% else %} + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" - {{ answer_choices[1] }} - - {% endif %}' + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: better_A_B + original_task: false + name: single_sentence_good_yes_no reference: '' - 9c3f75ab-e93c-45f5-8f4c-49c6c124c365: !Template - answer_choices: 1 ||| 2 - id: 9c3f75ab-e93c-45f5-8f4c-49c6c124c365 + dc37ec11-c672-4fd7-a224-cb788e185ec6: !Template + answer_choices: Yes ||| No + id: dc37ec11-c672-4fd7-a224-cb788e185ec6 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. - - {% if shuffled_order == 0 %} - - 1: {{ sentence_good }} - - 2: {{ sentence_bad }} - - {% else %} - - 1: {{ sentence_bad }} - - 2: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} + {% set shuffled_order_options = [0, 1] | random %} - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} - - {% endif %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: grammatical_which_one_1_2 - reference: '' - 9c989643-08c2-462d-9648-9f5232eccd8c: !Template - answer_choices: Yes ||| No - id: 9c989643-08c2-462d-9648-9f5232eccd8c - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_good_no_yes + name: single_sentence_bad_yes_no reference: '' - b3062ae3-0863-4f21-8e39-fa9401c4e72f: !Template + dc98c524-ca56-4650-b93e-38d2f142f000: !Template answer_choices: A ||| B - id: b3062ae3-0863-4f21-8e39-fa9401c4e72f + id: dc98c524-ca56-4650-b93e-38d2f142f000 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} @@ -247,47 +216,35 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B - reference: '' - d4b9c969-2d13-484a-b861-d8c7466c1c3b: !Template - answer_choices: Yes ||| No - id: d4b9c969-2d13-484a-b861-d8c7466c1c3b - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes + name: grammatical_which_one_A_B reference: '' - daf254e1-8a32-479b-a572-1f96ba37875e: !Template - answer_choices: A ||| B - id: daf254e1-8a32-479b-a572-1f96ba37875e + ec4db392-fe08-4c54-977f-fc53625374fb: !Template + answer_choices: 1 ||| 2 + id: ec4db392-fe08-4c54-977f-fc53625374fb jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -301,21 +258,5 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A - reference: '' - fafa88ee-fce7-4858-bd61-3510fe2cdabf: !Template - answer_choices: Yes ||| No - id: fafa88ee-fce7-4858-bd61-3510fe2cdabf - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no + name: grammatical_which_one_1_2 reference: '' diff --git a/promptsource/templates/blimp/tough_vs_raising_1/templates.yaml b/promptsource/templates/blimp/tough_vs_raising_1/templates.yaml index 392e1f8d7..12f9c0da1 100644 --- a/promptsource/templates/blimp/tough_vs_raising_1/templates.yaml +++ b/promptsource/templates/blimp/tough_vs_raising_1/templates.yaml @@ -1,23 +1,35 @@ dataset: blimp subset: tough_vs_raising_1 templates: - 08e32861-a173-46ee-9dc2-8a0552d1579c: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 08e32861-a173-46ee-9dc2-8a0552d1579c - jinja: ' ||| {{ sentence_good }}' + 1a0874ea-9df5-4b93-a61d-2dedab8625bb: !Template + answer_choices: Yes ||| No + id: 1a0874ea-9df5-4b93-a61d-2dedab8625bb + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata - choices_in_prompt: false + choices_in_prompt: true metrics: - Accuracy - original_task: true - name: true_null_prompt + original_task: false + name: single_sentence_bad_yes_no reference: '' - 23644448-b1d3-4c1b-81a5-98fdb5d96c12: !Template + 1a96fecf-fb39-438a-b1dc-1dd0544203ac: !Template answer_choices: 1 ||| 2 - id: 23644448-b1d3-4c1b-81a5-98fdb5d96c12 + id: 1a96fecf-fb39-438a-b1dc-1dd0544203ac jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} @@ -51,28 +63,35 @@ templates: original_task: true name: grammatical_which_one_1_2 reference: '' - 29f259ac-0258-4427-a777-27c198dd1d06: !Template + 45b19636-cde2-4ce7-a036-d2b49608852c: !Template answer_choices: Yes ||| No - id: 29f259ac-0258-4427-a777-27c198dd1d06 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + id: 45b19636-cde2-4ce7-a036-d2b49608852c + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_good_no_yes + name: single_sentence_good_yes_no reference: '' - 3846e1e0-b475-407c-bbad-08c966f66665: !Template + 5009927d-1b0d-4aba-89a2-3934e28c9989: !Template answer_choices: A ||| B - id: 3846e1e0-b475-407c-bbad-08c966f66665 + id: 5009927d-1b0d-4aba-89a2-3934e28c9989 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} @@ -106,55 +125,15 @@ templates: original_task: true name: grammatical_between_A_B reference: '' - 3a14af17-ec8f-408e-981a-af88a8712d36: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 3a14af17-ec8f-408e-981a-af88a8712d36 - jinja: '" ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: null_prompt - reference: '' - 403e805b-74d5-4869-b8c8-ea0a457d79ec: !Template - answer_choices: Yes ||| No - id: 403e805b-74d5-4869-b8c8-ea0a457d79ec - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no - reference: '' - 7aa3d847-b2c4-4928-8c57-29ac8e4ef72a: !Template - answer_choices: Yes ||| No - id: 7aa3d847-b2c4-4928-8c57-29ac8e4ef72a - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes - reference: '' - 9b76dce4-2c4f-4245-8a11-42e3a66fd05b: !Template + 6814638e-90c0-42fb-8cdc-5f0d3476bb74: !Template answer_choices: 1 ||| 2 - id: 9b76dce4-2c4f-4245-8a11-42e3a66fd05b + id: 6814638e-90c0-42fb-8cdc-5f0d3476bb74 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} @@ -188,12 +167,26 @@ templates: original_task: true name: grammatical_between_1_2 reference: '' - c2340ff3-cd1b-45d7-a976-ba64d86eada9: !Template + baea91cd-56b7-4257-8590-39f468fe318d: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: baea91cd-56b7-4257-8590-39f468fe318d + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + d39f8df5-0280-4984-824e-776346bd0f12: !Template answer_choices: A ||| B - id: c2340ff3-cd1b-45d7-a976-ba64d86eada9 + id: d39f8df5-0280-4984-824e-776346bd0f12 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} @@ -227,48 +220,12 @@ templates: original_task: true name: grammatical_which_one_A_B reference: '' - c2b75382-b113-4a3a-8175-68cccf5fce32: !Template + f6046c53-3199-4e2d-82cc-e4dcfe733181: !Template answer_choices: A ||| B - id: c2b75382-b113-4a3a-8175-68cccf5fce32 + id: f6046c53-3199-4e2d-82cc-e4dcfe733181 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. - - {% if shuffled_order == 0 %} - - Sentence A: {{ sentence_good }} - - Sentence B: {{ sentence_bad }} - - {% else %} - - Sentence A: {{ sentence_bad }} - - Sentence B: {{ sentence_good }} - - {% endif %} - - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} - - {% endif %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: better_B_A - reference: '' - c4b2a526-4671-4905-9419-837a132de34e: !Template - answer_choices: A ||| B - id: c4b2a526-4671-4905-9419-837a132de34e - jinja: '{% set shuffled_order = [0, 1] | random %} + {% set shuffled_order_options = [0, 1] | random %} Read the following two sentences. @@ -286,8 +243,8 @@ templates: {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -303,19 +260,3 @@ templates: original_task: true name: better_A_B reference: '' - dd2f5326-543d-42f2-b36f-50b9496ba609: !Template - answer_choices: Yes ||| No - id: dd2f5326-543d-42f2-b36f-50b9496ba609 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no - reference: '' diff --git a/promptsource/templates/blimp/tough_vs_raising_2/templates.yaml b/promptsource/templates/blimp/tough_vs_raising_2/templates.yaml index 26a442a65..644536f34 100644 --- a/promptsource/templates/blimp/tough_vs_raising_2/templates.yaml +++ b/promptsource/templates/blimp/tough_vs_raising_2/templates.yaml @@ -1,12 +1,15 @@ dataset: blimp subset: tough_vs_raising_2 templates: - 0a7ada73-c54a-4742-b4ce-b2595cc8e45a: !Template + 31a0e88f-de15-450f-aa80-afc02eb1f6a7: !Template answer_choices: A ||| B - id: 0a7ada73-c54a-4742-b4ce-b2595cc8e45a + id: 31a0e88f-de15-450f-aa80-afc02eb1f6a7 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} @@ -40,122 +43,67 @@ templates: original_task: true name: grammatical_which_one_A_B reference: '' - 11e9a98d-07bd-4c4a-aaa5-7805a0208dd9: !Template - answer_choices: 1 ||| 2 - id: 11e9a98d-07bd-4c4a-aaa5-7805a0208dd9 + 42615eed-d287-47f6-88c8-f366ad6350ce: !Template + answer_choices: Yes ||| No + id: 42615eed-d287-47f6-88c8-f366ad6350ce jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. - - {% if shuffled_order == 0 %} - - 1: {{ sentence_good }} - - 2: {{ sentence_bad }} - - {% else %} - - 1: {{ sentence_bad }} - - 2: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} + {% set shuffled_order_options = [0, 1] | random %} - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} - - {% endif %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: grammatical_between_1_2 - reference: '' - 45afaf9f-f6c7-4644-883c-680ba8b5a742: !Template - answer_choices: Yes ||| No - id: 45afaf9f-f6c7-4644-883c-680ba8b5a742 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_good_yes_no + name: single_sentence_bad_yes_no reference: '' - 6570c687-48d8-4f48-976e-6e6892d00ec1: !Template + 45838379-3e45-49cb-bb40-024291aa3ac7: !Template answer_choices: Yes ||| No - id: 6570c687-48d8-4f48-976e-6e6892d00ec1 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + id: 45838379-3e45-49cb-bb40-024291aa3ac7 + jinja: '{% set shuffled_order = [0, 1] | random %} - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes - reference: '' - 66ba5409-e49f-4259-ab8a-cd0f14a51122: !Template - answer_choices: Yes ||| No - id: 66ba5409-e49f-4259-ab8a-cd0f14a51122 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + {% set shuffled_order_options = [0, 1] | random %} - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_bad_yes_no - reference: '' - 6a27f6de-42c5-458d-b0fb-661ae16b37d9: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 6a27f6de-42c5-458d-b0fb-661ae16b37d9 - jinja: '" ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: null_prompt + name: single_sentence_good_yes_no reference: '' - a572d598-74b4-4f6c-ba1e-57c2961d34e4: !Template - answer_choices: A ||| B - id: a572d598-74b4-4f6c-ba1e-57c2961d34e4 + 77c255ac-eb29-4b5c-84ff-829ddd6d17ef: !Template + answer_choices: 1 ||| 2 + id: 77c255ac-eb29-4b5c-84ff-829ddd6d17ef jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -175,13 +123,15 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: grammatical_between_1_2 reference: '' - a5a408a7-df55-4433-839c-1773a68c5c4e: !Template + b8acabf2-cc33-4283-a10e-b6d354d04a78: !Template answer_choices: A ||| B - id: a5a408a7-df55-4433-839c-1773a68c5c4e + id: b8acabf2-cc33-4283-a10e-b6d354d04a78 jinja: '{% set shuffled_order = [0, 1] | random %} + {% set shuffled_order_options = [0, 1] | random %} + Read the following two sentences. {% if shuffled_order == 0 %} @@ -198,8 +148,8 @@ templates: {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -215,12 +165,26 @@ templates: original_task: true name: better_A_B reference: '' - abcac683-f725-493d-8fdb-886a8ec9b173: !Template + d25542d3-3bad-4e3c-88bf-b513ece45759: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: d25542d3-3bad-4e3c-88bf-b513ece45759 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + e1cd87f2-cf74-481b-8fd2-2d5f24eb8585: !Template answer_choices: 1 ||| 2 - id: abcac683-f725-493d-8fdb-886a8ec9b173 + id: e1cd87f2-cf74-481b-8fd2-2d5f24eb8585 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} @@ -254,29 +218,33 @@ templates: original_task: true name: grammatical_which_one_1_2 reference: '' - d3328908-30b5-43f8-b7b6-e9554e9495a0: !Template + fd5ee495-2ddc-41cc-8e5d-d925de156a89: !Template answer_choices: A ||| B - id: d3328908-30b5-43f8-b7b6-e9554e9495a0 + id: fd5ee495-2ddc-41cc-8e5d-d925de156a89 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -290,32 +258,5 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A - reference: '' - d515fca1-feda-4cb8-9209-e42eb2319502: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: d515fca1-feda-4cb8-9209-e42eb2319502 - jinja: ' ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: true_null_prompt - reference: '' - f9905ae2-4060-410d-aa47-e075c4f5434a: !Template - answer_choices: Yes ||| No - id: f9905ae2-4060-410d-aa47-e075c4f5434a - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes + name: grammatical_between_A_B reference: '' diff --git a/promptsource/templates/blimp/transitive/templates.yaml b/promptsource/templates/blimp/transitive/templates.yaml index 47dfc48cd..c1e2f3487 100644 --- a/promptsource/templates/blimp/transitive/templates.yaml +++ b/promptsource/templates/blimp/transitive/templates.yaml @@ -1,69 +1,31 @@ dataset: blimp subset: transitive templates: - 0eba4768-cb21-40cd-9bdd-db9e6eb2e1a3: !Template - answer_choices: 1 ||| 2 - id: 0eba4768-cb21-40cd-9bdd-db9e6eb2e1a3 - jinja: '{% set shuffled_order = [0, 1] | random %} - - Which one of the following sentences is grammatical? Please answer 1 or 2. - - {% if shuffled_order == 0 %} - - 1: {{ sentence_good }} - - 2: {{ sentence_bad }} - - {% else %} - - 1: {{ sentence_bad }} - - 2: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} - - {% endif %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: grammatical_which_one_1_2 - reference: '' - 5f50ba61-41f0-4a65-a36a-54870101dc27: !Template + 1c182220-eb99-4505-9d06-3cd032dee4d5: !Template answer_choices: A ||| B - id: 5f50ba61-41f0-4a65-a36a-54870101dc27 + id: 1c182220-eb99-4505-9d06-3cd032dee4d5 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -77,41 +39,17 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B - reference: '' - 62dc4b8e-5c20-407a-8f1b-417197081e31: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 62dc4b8e-5c20-407a-8f1b-417197081e31 - jinja: '" ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: null_prompt - reference: '' - 6a1c35fa-4ef4-49bf-a5a2-62055468e338: !Template - answer_choices: Yes ||| No - id: 6a1c35fa-4ef4-49bf-a5a2-62055468e338 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no + name: better_A_B reference: '' - 6b66dc4d-8eeb-4b1b-94eb-fb50bb9502d7: !Template + 49c4bd4b-43aa-4020-8ec8-cc7a6644be19: !Template answer_choices: 1 ||| 2 - id: 6b66dc4d-8eeb-4b1b-94eb-fb50bb9502d7 + id: 49c4bd4b-43aa-4020-8ec8-cc7a6644be19 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} @@ -143,30 +81,17 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 - reference: '' - 6e0ee564-add5-4cac-8217-858f910f8d03: !Template - answer_choices: Yes ||| No - id: 6e0ee564-add5-4cac-8217-858f910f8d03 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no + name: grammatical_which_one_1_2 reference: '' - acebb0cb-612c-4cad-a40a-d2c7b5c7cdb6: !Template + 4aed6b5d-b048-4715-8cac-3bc4b407e82d: !Template answer_choices: A ||| B - id: acebb0cb-612c-4cad-a40a-d2c7b5c7cdb6 + id: 4aed6b5d-b048-4715-8cac-3bc4b407e82d jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} @@ -200,56 +125,64 @@ templates: original_task: true name: grammatical_which_one_A_B reference: '' - c15255e9-83fe-4b83-81d3-4894ab7d366f: !Template + 4f7de6ef-e0fd-4d6d-8537-8b32af44a86d: !Template answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: c15255e9-83fe-4b83-81d3-4894ab7d366f + id: 4f7de6ef-e0fd-4d6d-8537-8b32af44a86d jinja: ' ||| {{ sentence_good }}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - Accuracy original_task: true - name: true_null_prompt + name: null_prompt reference: '' - d2e5a8a9-8b38-400b-8dff-4198560eac9b: !Template + 8c770286-45b2-461a-974b-48ef0cb7d0ac: !Template answer_choices: Yes ||| No - id: d2e5a8a9-8b38-400b-8dff-4198560eac9b - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + id: 8c770286-45b2-461a-974b-48ef0cb7d0ac + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_good_no_yes + name: single_sentence_bad_yes_no reference: '' - d7f89c1a-8780-4c5b-8928-73ae6763d36b: !Template - answer_choices: A ||| B - id: d7f89c1a-8780-4c5b-8928-73ae6763d36b + 9b8f0825-1565-439c-8f3d-3d620898e948: !Template + answer_choices: 1 ||| 2 + id: 9b8f0825-1565-439c-8f3d-3d620898e948 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -263,47 +196,35 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B - reference: '' - ea92d1bb-5e82-41ed-bade-dca54a0f6f70: !Template - answer_choices: Yes ||| No - id: ea92d1bb-5e82-41ed-bade-dca54a0f6f70 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes + name: grammatical_between_1_2 reference: '' - ff10bbd1-3a44-42af-988d-7e45cc216750: !Template + b688ecf2-b95a-499f-b20f-c8a9e8305e22: !Template answer_choices: A ||| B - id: ff10bbd1-3a44-42af-988d-7e45cc216750 + id: b688ecf2-b95a-499f-b20f-c8a9e8305e22 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -317,5 +238,25 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: grammatical_between_A_B + reference: '' + f04f5a5b-5243-469b-8ff3-8edce3c4bcd8: !Template + answer_choices: Yes ||| No + id: f04f5a5b-5243-469b-8ff3-8edce3c4bcd8 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no reference: '' diff --git a/promptsource/templates/blimp/wh_island/templates.yaml b/promptsource/templates/blimp/wh_island/templates.yaml index 88bdf586e..e9aae6f22 100644 --- a/promptsource/templates/blimp/wh_island/templates.yaml +++ b/promptsource/templates/blimp/wh_island/templates.yaml @@ -1,24 +1,27 @@ dataset: blimp subset: wh_island templates: - 2bca2bb6-1bb1-4c51-985a-d85f1c6060c4: !Template - answer_choices: 1 ||| 2 - id: 2bca2bb6-1bb1-4c51-985a-d85f1c6060c4 + 0be4ddb1-91d2-46e7-92ac-ba864eb83a6b: !Template + answer_choices: A ||| B + id: 0be4ddb1-91d2-46e7-92ac-ba864eb83a6b jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -38,57 +41,48 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 - reference: '' - 2d3c3858-66c1-45b3-ab88-1c21fd34976f: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 2d3c3858-66c1-45b3-ab88-1c21fd34976f - jinja: ' ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: true_null_prompt + name: grammatical_which_one_A_B reference: '' - 2fd453fc-36de-4441-b4d5-93ba518e3de1: !Template + 1354e34b-6ac5-4032-8cd0-329a76b24a11: !Template answer_choices: Yes ||| No - id: 2fd453fc-36de-4441-b4d5-93ba518e3de1 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + id: 1354e34b-6ac5-4032-8cd0-329a76b24a11 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_good_no_yes + name: single_sentence_good_yes_no reference: '' - 4adeda8c-a1ce-45bf-8435-2a76c4eb3375: !Template - answer_choices: Yes ||| No - id: 4adeda8c-a1ce-45bf-8435-2a76c4eb3375 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' + 81ca64d6-7bed-46ee-9c5a-b9c3395db11d: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 81ca64d6-7bed-46ee-9c5a-b9c3395db11d + jinja: ' ||| {{ sentence_good }}' metadata: !TemplateMetadata - choices_in_prompt: true + choices_in_prompt: false metrics: - Accuracy - original_task: false - name: single_sentence_bad_yes_no + original_task: true + name: null_prompt reference: '' - 54087567-96f7-4573-93ca-d1ca0b3bdee5: !Template + 891da904-88cb-44c5-9a1a-90ad0842b316: !Template answer_choices: A ||| B - id: 54087567-96f7-4573-93ca-d1ca0b3bdee5 + id: 891da904-88cb-44c5-9a1a-90ad0842b316 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} @@ -122,50 +116,15 @@ templates: original_task: true name: grammatical_between_A_B reference: '' - 6508efe4-f064-4f47-84d3-a6d4fa289b18: !Template - answer_choices: A ||| B - id: 6508efe4-f064-4f47-84d3-a6d4fa289b18 - jinja: '{% set shuffled_order = [0, 1] | random %} - - Read the following two sentences. - - {% if shuffled_order == 0 %} - - Sentence A: {{ sentence_good }} - - Sentence B: {{ sentence_bad }} - - {% else %} - - Sentence A: {{ sentence_bad }} - - Sentence B: {{ sentence_good }} - - {% endif %} - - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} - - {% endif %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: better_B_A - reference: '' - 74b49b39-8cd0-4348-a0dc-5e6b46720c43: !Template + 8a8d6781-67b5-46ed-8ebc-0af0e6cb304b: !Template answer_choices: 1 ||| 2 - id: 74b49b39-8cd0-4348-a0dc-5e6b46720c43 + id: 8a8d6781-67b5-46ed-8ebc-0af0e6cb304b jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} @@ -199,46 +158,31 @@ templates: original_task: true name: grammatical_which_one_1_2 reference: '' - 86c69941-3c5c-444d-a753-3d988d7c5a24: !Template - answer_choices: Yes ||| No - id: 86c69941-3c5c-444d-a753-3d988d7c5a24 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes - reference: '' - 8799761e-cfb5-4054-93be-0c13781120fb: !Template + bbd8dbcb-7755-4444-bd40-2a9bdb2d5206: !Template answer_choices: A ||| B - id: 8799761e-cfb5-4054-93be-0c13781120fb + id: bbd8dbcb-7755-4444-bd40-2a9bdb2d5206 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -252,42 +196,35 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B - reference: '' - 8dd56f62-03c2-4187-b09a-f259c1c27aa2: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 8dd56f62-03c2-4187-b09a-f259c1c27aa2 - jinja: '" ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: null_prompt + name: better_A_B reference: '' - e3100602-a400-41b6-ab3f-2e8c0cac93ee: !Template - answer_choices: A ||| B - id: e3100602-a400-41b6-ab3f-2e8c0cac93ee + ebf77652-f993-43e0-b6b8-3c87e7b2a85e: !Template + answer_choices: 1 ||| 2 + id: ebf77652-f993-43e0-b6b8-3c87e7b2a85e jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -301,21 +238,25 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: grammatical_between_1_2 reference: '' - ea6793cc-cb1b-4fd0-a79b-c649991add1c: !Template + ec1f1fb2-dacb-44f8-8870-5cb9f82b4ef2: !Template answer_choices: Yes ||| No - id: ea6793cc-cb1b-4fd0-a79b-c649991add1c - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + id: ec1f1fb2-dacb-44f8-8870-5cb9f82b4ef2 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_good_yes_no + name: single_sentence_bad_yes_no reference: '' diff --git a/promptsource/templates/blimp/wh_questions_object_gap/templates.yaml b/promptsource/templates/blimp/wh_questions_object_gap/templates.yaml index e41eab632..b7958341d 100644 --- a/promptsource/templates/blimp/wh_questions_object_gap/templates.yaml +++ b/promptsource/templates/blimp/wh_questions_object_gap/templates.yaml @@ -1,40 +1,27 @@ dataset: blimp subset: wh_questions_object_gap templates: - 1275a82f-a19d-4fed-858d-2eb27c6472ba: !Template - answer_choices: Yes ||| No - id: 1275a82f-a19d-4fed-858d-2eb27c6472ba - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no - reference: '' - 296af7e5-b024-47cd-b52d-98b8abce3ce9: !Template - answer_choices: 1 ||| 2 - id: 296af7e5-b024-47cd-b52d-98b8abce3ce9 + 40d31ad1-13e4-4ce3-b6eb-e1614ccaf11b: !Template + answer_choices: A ||| B + id: 40d31ad1-13e4-4ce3-b6eb-e1614ccaf11b jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -54,31 +41,35 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: grammatical_between_A_B reference: '' - 2c46a0f3-ccee-486d-9153-2d1409ba3061: !Template + 536d5978-86d4-4146-a51f-e64cfbb8edc2: !Template answer_choices: A ||| B - id: 2c46a0f3-ccee-486d-9153-2d1409ba3061 + id: 536d5978-86d4-4146-a51f-e64cfbb8edc2 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -92,59 +83,33 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A - reference: '' - 4f7c3aee-750b-4569-8d2f-a35d8b240f96: !Template - answer_choices: Yes ||| No - id: 4f7c3aee-750b-4569-8d2f-a35d8b240f96 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes - reference: '' - 68365a2c-b632-4905-98c1-6fefe801103b: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 68365a2c-b632-4905-98c1-6fefe801103b - jinja: '" ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: null_prompt + name: grammatical_which_one_A_B reference: '' - 6bd5fce7-9e2c-4f82-bfe6-9229f1738260: !Template - answer_choices: 1 ||| 2 - id: 6bd5fce7-9e2c-4f82-bfe6-9229f1738260 + 560797ab-b625-4758-aec6-bde81ef69cb7: !Template + answer_choices: A ||| B + id: 560797ab-b625-4758-aec6-bde81ef69cb7 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -158,42 +123,40 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: better_A_B reference: '' - 7925636e-0693-42ab-825e-35bdc44350c9: !Template - answer_choices: Yes ||| No - id: 7925636e-0693-42ab-825e-35bdc44350c9 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' + a2b50b51-922d-49a0-a84d-510665fb14d0: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: a2b50b51-922d-49a0-a84d-510665fb14d0 + jinja: ' ||| {{ sentence_good }}' metadata: !TemplateMetadata - choices_in_prompt: true + choices_in_prompt: false metrics: - Accuracy - original_task: false - name: single_sentence_bad_yes_no + original_task: true + name: null_prompt reference: '' - 83a017c4-405f-40b7-9b33-bc2219c4136b: !Template - answer_choices: A ||| B - id: 83a017c4-405f-40b7-9b33-bc2219c4136b + b54be2ff-4e50-4497-af1f-f7281db8aa77: !Template + answer_choices: 1 ||| 2 + id: b54be2ff-4e50-4497-af1f-f7281db8aa77 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -213,97 +176,75 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: grammatical_between_1_2 reference: '' - 8c61d74b-b230-42c4-8cf6-60a92288eaeb: !Template + ccfd0ff1-247e-4c39-9157-81275a5d5f77: !Template answer_choices: Yes ||| No - id: 8c61d74b-b230-42c4-8cf6-60a92288eaeb - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + id: ccfd0ff1-247e-4c39-9157-81275a5d5f77 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_good_no_yes + name: single_sentence_good_yes_no reference: '' - b76f0d57-1881-447b-88f9-44b1fadf8399: !Template - answer_choices: A ||| B - id: b76f0d57-1881-447b-88f9-44b1fadf8399 + e1219361-709d-467d-aeaf-5733e53e607e: !Template + answer_choices: Yes ||| No + id: e1219361-709d-467d-aeaf-5733e53e607e jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. - - {% if shuffled_order == 0 %} - - A: {{ sentence_good }} + {% set shuffled_order_options = [0, 1] | random %} - B: {{ sentence_bad }} - - {% else %} + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" - A: {{ sentence_bad }} - - B: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} - - {% endif %}' + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: grammatical_between_A_B - reference: '' - c9c0261f-97c4-4cec-941c-b241a09bf67c: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: c9c0261f-97c4-4cec-941c-b241a09bf67c - jinja: ' ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: true_null_prompt + original_task: false + name: single_sentence_bad_yes_no reference: '' - d65a937f-64f8-4353-b166-0b0540905e55: !Template - answer_choices: A ||| B - id: d65a937f-64f8-4353-b166-0b0540905e55 + ff4863c5-c8b4-476c-bc3c-a138bb226126: !Template + answer_choices: 1 ||| 2 + id: ff4863c5-c8b4-476c-bc3c-a138bb226126 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -317,5 +258,5 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: grammatical_which_one_1_2 reference: '' diff --git a/promptsource/templates/blimp/wh_questions_subject_gap/templates.yaml b/promptsource/templates/blimp/wh_questions_subject_gap/templates.yaml index 8dbf066f6..56daffcba 100644 --- a/promptsource/templates/blimp/wh_questions_subject_gap/templates.yaml +++ b/promptsource/templates/blimp/wh_questions_subject_gap/templates.yaml @@ -1,51 +1,26 @@ dataset: blimp subset: wh_questions_subject_gap templates: - 0126b33c-f9a6-4bd0-81df-b6da0634a05b: !Template - answer_choices: A ||| B - id: 0126b33c-f9a6-4bd0-81df-b6da0634a05b - jinja: '{% set shuffled_order = [0, 1] | random %} - - Which one of the following sentences is grammatical? Please answer A or B. - - {% if shuffled_order == 0 %} - - A: {{ sentence_good }} - - B: {{ sentence_bad }} - - {% else %} - - A: {{ sentence_bad }} - - B: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} - - {% endif %}' + 04572431-8a02-4435-b06e-a1ffe47d1bc2: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 04572431-8a02-4435-b06e-a1ffe47d1bc2 + jinja: ' ||| {{ sentence_good }}' metadata: !TemplateMetadata - choices_in_prompt: true + choices_in_prompt: false metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: null_prompt reference: '' - 16570efc-68bf-4157-b82c-dab803929cea: !Template + 0bad60f2-5b52-4a99-a486-a4bcbd7839a5: !Template answer_choices: 1 ||| 2 - id: 16570efc-68bf-4157-b82c-dab803929cea + id: 0bad60f2-5b52-4a99-a486-a4bcbd7839a5 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} @@ -77,58 +52,35 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 - reference: '' - 39547bc9-35e7-40d2-9bae-fe3b91677ef9: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 39547bc9-35e7-40d2-9bae-fe3b91677ef9 - jinja: ' ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: true_null_prompt - reference: '' - 474feff6-8c2f-40af-9a50-07304f248440: !Template - answer_choices: Yes ||| No - id: 474feff6-8c2f-40af-9a50-07304f248440 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no + name: grammatical_between_1_2 reference: '' - 4c021b8e-1135-4dcc-b11d-ce18fab28f02: !Template + 0d64911f-41f6-4305-831b-f8e2e3474068: !Template answer_choices: A ||| B - id: 4c021b8e-1135-4dcc-b11d-ce18fab28f02 + id: 0d64911f-41f6-4305-831b-f8e2e3474068 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -142,26 +94,29 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: grammatical_which_one_A_B reference: '' - 5190dcc5-c105-4c4e-831f-fe6b8fe3ad1f: !Template - answer_choices: 1 ||| 2 - id: 5190dcc5-c105-4c4e-831f-fe6b8fe3ad1f + 1d281f37-5b87-43f9-94c3-bcc0e2d0f89d: !Template + answer_choices: A ||| B + id: 1d281f37-5b87-43f9-94c3-bcc0e2d0f89d jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -181,32 +136,33 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: grammatical_between_A_B reference: '' - a419a20a-e200-4c3f-822b-3293807164a1: !Template + 781a0ca9-63ad-4d94-ae66-388246dabffe: !Template answer_choices: A ||| B - id: a419a20a-e200-4c3f-822b-3293807164a1 + id: 781a0ca9-63ad-4d94-ae66-388246dabffe jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -220,90 +176,55 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: better_A_B reference: '' - a6af6beb-c9d9-42c0-8ac8-20a980f97328: !Template + b2c97636-2bf7-429e-9af7-5186b5ea0e46: !Template answer_choices: Yes ||| No - id: a6af6beb-c9d9-42c0-8ac8-20a980f97328 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + id: b2c97636-2bf7-429e-9af7-5186b5ea0e46 + jinja: '{% set shuffled_order = [0, 1] | random %} - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no - reference: '' - af764f44-c852-4a97-bcd2-efb0dc8d9fac: !Template - answer_choices: Yes ||| No - id: af764f44-c852-4a97-bcd2-efb0dc8d9fac - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + {% set shuffled_order_options = [0, 1] | random %} - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes - reference: '' - b57011b1-bdaa-4274-a2ce-be4a9bcd62f5: !Template - answer_choices: Yes ||| No - id: b57011b1-bdaa-4274-a2ce-be4a9bcd62f5 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_good_no_yes - reference: '' - eb6f0983-bf35-4d1c-9f32-17a986fc1374: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: eb6f0983-bf35-4d1c-9f32-17a986fc1374 - jinja: '" ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: null_prompt + name: single_sentence_bad_yes_no reference: '' - ef016243-c2eb-423f-a533-fcc60f3655fb: !Template - answer_choices: A ||| B - id: ef016243-c2eb-423f-a533-fcc60f3655fb + f4a1c41d-c3e1-4ddf-981c-cf4b8c8a9297: !Template + answer_choices: 1 ||| 2 + id: f4a1c41d-c3e1-4ddf-981c-cf4b8c8a9297 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -317,5 +238,25 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: grammatical_which_one_1_2 + reference: '' + f86d0901-0f03-452a-bfa6-b21e2b805171: !Template + answer_choices: Yes ||| No + id: f86d0901-0f03-452a-bfa6-b21e2b805171 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no reference: '' diff --git a/promptsource/templates/blimp/wh_questions_subject_gap_long_distance/templates.yaml b/promptsource/templates/blimp/wh_questions_subject_gap_long_distance/templates.yaml index 7099161f3..3129d3b02 100644 --- a/promptsource/templates/blimp/wh_questions_subject_gap_long_distance/templates.yaml +++ b/promptsource/templates/blimp/wh_questions_subject_gap_long_distance/templates.yaml @@ -1,50 +1,35 @@ dataset: blimp subset: wh_questions_subject_gap_long_distance templates: - 00d475f6-2219-4a2f-9b59-0af9790af10d: !Template - answer_choices: A ||| B - id: 00d475f6-2219-4a2f-9b59-0af9790af10d + 29f05465-4289-4ded-9863-92e36d90c71f: !Template + answer_choices: Yes ||| No + id: 29f05465-4289-4ded-9863-92e36d90c71f jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. - - {% if shuffled_order == 0 %} - - Sentence A: {{ sentence_good }} + {% set shuffled_order_options = [0, 1] | random %} - Sentence B: {{ sentence_bad }} + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" - {% else %} - - Sentence A: {{ sentence_bad }} - - Sentence B: {{ sentence_good }} - - {% endif %} - - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} - - {% endif %}' + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: better_B_A + original_task: false + name: single_sentence_bad_yes_no reference: '' - 4073e6da-b998-4523-af4a-2662455efba8: !Template + 42eb0ad3-388d-43ad-af2c-a91e8b78db99: !Template answer_choices: A ||| B - id: 4073e6da-b998-4523-af4a-2662455efba8 + id: 42eb0ad3-388d-43ad-af2c-a91e8b78db99 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} @@ -76,17 +61,21 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B + name: grammatical_between_A_B reference: '' - 6555c3e9-e408-475b-a828-e0fb864c0aa9: !Template + 4330ea36-6c68-44cf-b91e-bb00b34bf1fa: !Template answer_choices: Yes ||| No - id: 6555c3e9-e408-475b-a828-e0fb864c0aa9 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + id: 4330ea36-6c68-44cf-b91e-bb00b34bf1fa + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: @@ -94,12 +83,15 @@ templates: original_task: false name: single_sentence_good_yes_no reference: '' - 7484d736-9848-46b3-903a-5d603ee81987: !Template + 6105db7f-bb5a-48fc-a8e2-0f5bc1034797: !Template answer_choices: 1 ||| 2 - id: 7484d736-9848-46b3-903a-5d603ee81987 + id: 6105db7f-bb5a-48fc-a8e2-0f5bc1034797 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} @@ -131,26 +123,40 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: grammatical_between_1_2 reference: '' - 85d4c1ca-20bf-47e0-89d3-4289d05357d8: !Template - answer_choices: 1 ||| 2 - id: 85d4c1ca-20bf-47e0-89d3-4289d05357d8 + 63bbab26-1eb1-4db0-bdb3-0c3b3e7a5ef9: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 63bbab26-1eb1-4db0-bdb3-0c3b3e7a5ef9 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + a281eb95-2ea6-4425-8344-d93598bfe1c7: !Template + answer_choices: A ||| B + id: a281eb95-2ea6-4425-8344-d93598bfe1c7 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -170,37 +176,29 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 - reference: '' - 89d1c1ea-568e-4b48-9771-2396af2f3f1a: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 89d1c1ea-568e-4b48-9771-2396af2f3f1a - jinja: ' ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: true_null_prompt + name: grammatical_which_one_A_B reference: '' - ad11e240-e8ea-4a4a-8413-a30589831045: !Template - answer_choices: A ||| B - id: ad11e240-e8ea-4a4a-8413-a30589831045 + e8d445ca-9f07-48d4-8786-173bbf7319e1: !Template + answer_choices: 1 ||| 2 + id: e8d445ca-9f07-48d4-8786-173bbf7319e1 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -220,13 +218,15 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: grammatical_which_one_1_2 reference: '' - b33871dc-c6f9-4531-b09e-6ffb72d7bc21: !Template + ebb545b1-2e00-45a5-88ff-228d138cb157: !Template answer_choices: A ||| B - id: b33871dc-c6f9-4531-b09e-6ffb72d7bc21 + id: ebb545b1-2e00-45a5-88ff-228d138cb157 jinja: '{% set shuffled_order = [0, 1] | random %} + {% set shuffled_order_options = [0, 1] | random %} + Read the following two sentences. {% if shuffled_order == 0 %} @@ -243,8 +243,8 @@ templates: {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -260,62 +260,3 @@ templates: original_task: true name: better_A_B reference: '' - ca1ba3be-8c69-4d67-ba03-6ddb9fd148b3: !Template - answer_choices: Yes ||| No - id: ca1ba3be-8c69-4d67-ba03-6ddb9fd148b3 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes - reference: '' - dc90181a-2866-42ce-864b-7330ec839234: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: dc90181a-2866-42ce-864b-7330ec839234 - jinja: '" ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: null_prompt - reference: '' - e985f87a-6a81-4df7-b705-fa090af87c21: !Template - answer_choices: Yes ||| No - id: e985f87a-6a81-4df7-b705-fa090af87c21 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes - reference: '' - fe81c28a-0587-4f3f-8613-081b2cee2efe: !Template - answer_choices: Yes ||| No - id: fe81c28a-0587-4f3f-8613-081b2cee2efe - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no - reference: '' diff --git a/promptsource/templates/blimp/wh_vs_that_no_gap/templates.yaml b/promptsource/templates/blimp/wh_vs_that_no_gap/templates.yaml index 9397055c9..7f308b84a 100644 --- a/promptsource/templates/blimp/wh_vs_that_no_gap/templates.yaml +++ b/promptsource/templates/blimp/wh_vs_that_no_gap/templates.yaml @@ -1,10 +1,10 @@ dataset: blimp subset: wh_vs_that_no_gap templates: - 03861afc-89b4-4b90-9d5a-167c38df6cf2: !Template + 1967f46d-e4fd-4cb6-94c6-692d6d7f0d95: !Template answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 03861afc-89b4-4b90-9d5a-167c38df6cf2 - jinja: '" ||| {{ sentence_good }}' + id: 1967f46d-e4fd-4cb6-94c6-692d6d7f0d95 + jinja: ' ||| {{ sentence_good }}' metadata: !TemplateMetadata choices_in_prompt: false metrics: @@ -12,29 +12,33 @@ templates: original_task: true name: null_prompt reference: '' - 1a950cbd-dea6-41be-ab64-29034549bbf9: !Template - answer_choices: A ||| B - id: 1a950cbd-dea6-41be-ab64-29034549bbf9 + 86dce9cf-2a28-41f8-8e37-0ed47dc8504e: !Template + answer_choices: 1 ||| 2 + id: 86dce9cf-2a28-41f8-8e37-0ed47dc8504e jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -48,32 +52,33 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A + name: grammatical_between_1_2 reference: '' - 1f0b1a4b-20c6-4217-8d64-8acfc1413b6d: !Template + 8d29bf78-af7c-4b5f-b0a3-f001de1b17dc: !Template answer_choices: A ||| B - id: 1f0b1a4b-20c6-4217-8d64-8acfc1413b6d + id: 8d29bf78-af7c-4b5f-b0a3-f001de1b17dc jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -87,58 +92,29 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B - reference: '' - 5d0c1b60-c6cb-480d-8c82-c1bacc3eaec8: !Template - answer_choices: Yes ||| No - id: 5d0c1b60-c6cb-480d-8c82-c1bacc3eaec8 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no - reference: '' - 5eb1161b-df0b-4ff4-b4c8-c29456ed2481: !Template - answer_choices: Yes ||| No - id: 5eb1161b-df0b-4ff4-b4c8-c29456ed2481 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes + name: better_A_B reference: '' - 64d71314-cde2-45d9-a782-7cf28a502a62: !Template - answer_choices: 1 ||| 2 - id: 64d71314-cde2-45d9-a782-7cf28a502a62 + 92d37871-6182-4ae3-b853-2135d71c53c5: !Template + answer_choices: A ||| B + id: 92d37871-6182-4ae3-b853-2135d71c53c5 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -158,70 +134,75 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: grammatical_which_one_A_B reference: '' - 7221e9ee-e560-4b98-b263-7042c51b84a1: !Template - answer_choices: 1 ||| 2 - id: 7221e9ee-e560-4b98-b263-7042c51b84a1 + 968d8473-ffbe-4fc9-bcf4-c3f1cee02274: !Template + answer_choices: Yes ||| No + id: 968d8473-ffbe-4fc9-bcf4-c3f1cee02274 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} - {% if shuffled_order == 0 %} + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" - 1: {{ sentence_good }} - - 2: {{ sentence_bad }} - - {% else %} - - 1: {{ sentence_bad }} - - 2: {{ sentence_good }} - - {% endif %} - - ||| - - {% if shuffled_order == 0 %} - - {{ answer_choices[0] }} + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 99b5cdd0-67b6-47ac-93d7-bc5dcd54c54b: !Template + answer_choices: Yes ||| No + id: 99b5cdd0-67b6-47ac-93d7-bc5dcd54c54b + jinja: '{% set shuffled_order = [0, 1] | random %} - {% else %} + {% set shuffled_order_options = [0, 1] | random %} - {{ answer_choices[1] }} + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" - {% endif %}' + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: grammatical_which_one_1_2 + original_task: false + name: single_sentence_good_yes_no reference: '' - b00be7cf-766c-4268-97eb-b04ecb10b565: !Template - answer_choices: A ||| B - id: b00be7cf-766c-4268-97eb-b04ecb10b565 + dc7c7d5d-c532-4828-bb37-102bf1331b2f: !Template + answer_choices: 1 ||| 2 + id: dc7c7d5d-c532-4828-bb37-102bf1331b2f jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -235,14 +216,17 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: grammatical_which_one_1_2 reference: '' - ba0cf74f-a1dc-4183-a573-4b6ea71b7bd0: !Template + e61e4273-9bdd-42f8-9339-ce07df2feaee: !Template answer_choices: A ||| B - id: ba0cf74f-a1dc-4183-a573-4b6ea71b7bd0 + id: e61e4273-9bdd-42f8-9339-ce07df2feaee jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} @@ -274,48 +258,5 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B - reference: '' - c298b934-e17a-4175-96d8-998e7669af42: !Template - answer_choices: Yes ||| No - id: c298b934-e17a-4175-96d8-998e7669af42 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes - reference: '' - e5315a89-f463-4fc9-94f3-04d59e96e36c: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: e5315a89-f463-4fc9-94f3-04d59e96e36c - jinja: ' ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: true_null_prompt - reference: '' - e8b23765-cc3d-4be1-a5fd-782fc6eeb195: !Template - answer_choices: Yes ||| No - id: e8b23765-cc3d-4be1-a5fd-782fc6eeb195 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no + name: grammatical_between_A_B reference: '' diff --git a/promptsource/templates/blimp/wh_vs_that_no_gap_long_distance/templates.yaml b/promptsource/templates/blimp/wh_vs_that_no_gap_long_distance/templates.yaml index 584999e19..a5ec43a5c 100644 --- a/promptsource/templates/blimp/wh_vs_that_no_gap_long_distance/templates.yaml +++ b/promptsource/templates/blimp/wh_vs_that_no_gap_long_distance/templates.yaml @@ -1,56 +1,27 @@ dataset: blimp subset: wh_vs_that_no_gap_long_distance templates: - 0354119f-5e9f-4787-a5ea-9411712261ec: !Template - answer_choices: Yes ||| No - id: 0354119f-5e9f-4787-a5ea-9411712261ec - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes - reference: '' - 1611776d-6dfc-4296-9ce5-ce72b0f4d9ed: !Template - answer_choices: Yes ||| No - id: 1611776d-6dfc-4296-9ce5-ce72b0f4d9ed - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no - reference: '' - 17fb8eac-e5e6-49c6-8b24-cd67a2472051: !Template - answer_choices: A ||| B - id: 17fb8eac-e5e6-49c6-8b24-cd67a2472051 + 3b6b4263-da7a-4e05-a50b-ecdb4421196b: !Template + answer_choices: 1 ||| 2 + id: 3b6b4263-da7a-4e05-a50b-ecdb4421196b jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -70,32 +41,53 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: grammatical_which_one_1_2 reference: '' - 1924cce9-dc96-4daf-9084-e548b1ae265c: !Template - answer_choices: 1 ||| 2 - id: 1924cce9-dc96-4daf-9084-e548b1ae265c + 4ee3c7ef-c1f3-4dd8-892d-d8d3d108f80c: !Template + answer_choices: Yes ||| No + id: 4ee3c7ef-c1f3-4dd8-892d-d8d3d108f80c jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 869e71d2-6236-4975-b6d1-838ef22a9dc4: !Template + answer_choices: A ||| B + id: 869e71d2-6236-4975-b6d1-838ef22a9dc4 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + Sentence A: {{ sentence_good }} - 2: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - 2: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -109,31 +101,35 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 + name: better_A_B reference: '' - 2b037c6f-7957-4362-a7c8-c2bae9552e6d: !Template + 93962115-2842-47ad-bfbd-bfc8a54ebc99: !Template answer_choices: A ||| B - id: 2b037c6f-7957-4362-a7c8-c2bae9552e6d + id: 93962115-2842-47ad-bfbd-bfc8a54ebc99 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + A: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + A: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -147,30 +143,17 @@ templates: metrics: - Accuracy original_task: true - name: better_B_A - reference: '' - 2f8466d6-5eb9-46d1-9fa1-99e80f589c6f: !Template - answer_choices: Yes ||| No - id: 2f8466d6-5eb9-46d1-9fa1-99e80f589c6f - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_no_yes + name: grammatical_between_A_B reference: '' - 49ff211c-81f7-49e9-8c6d-e47bf62059c9: !Template + 962c7019-9790-4204-86b5-bca38e94096c: !Template answer_choices: 1 ||| 2 - id: 49ff211c-81f7-49e9-8c6d-e47bf62059c9 + id: 962c7019-9790-4204-86b5-bca38e94096c jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} @@ -204,88 +187,46 @@ templates: original_task: true name: grammatical_between_1_2 reference: '' - 54b70e37-2e72-40c4-90f7-9d8bfc767ba3: !Template + d241c093-8d87-451f-85fc-2baf3296fb2f: !Template answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 54b70e37-2e72-40c4-90f7-9d8bfc767ba3 + id: d241c093-8d87-451f-85fc-2baf3296fb2f jinja: ' ||| {{ sentence_good }}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - Accuracy original_task: true - name: true_null_prompt + name: null_prompt reference: '' - 54f00e83-11ed-403f-9dda-6c4284948004: !Template + e7a4dc29-364a-48d3-9987-ee19a232e2df: !Template answer_choices: Yes ||| No - id: 54f00e83-11ed-403f-9dda-6c4284948004 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no - reference: '' - 7889bb75-004c-46f0-b4fd-5c32ea98cb21: !Template - answer_choices: A ||| B - id: 7889bb75-004c-46f0-b4fd-5c32ea98cb21 + id: e7a4dc29-364a-48d3-9987-ee19a232e2df jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. - - {% if shuffled_order == 0 %} + {% set shuffled_order_options = [0, 1] | random %} - Sentence A: {{ sentence_good }} + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" - Sentence B: {{ sentence_bad }} - - {% else %} - - Sentence A: {{ sentence_bad }} - - Sentence B: {{ sentence_good }} - - {% endif %} - - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} - - {% endif %}' + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: better_A_B - reference: '' - be848bb1-ffa6-4478-afb8-93bb284ad415: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: be848bb1-ffa6-4478-afb8-93bb284ad415 - jinja: '" ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: null_prompt + original_task: false + name: single_sentence_good_yes_no reference: '' - fec5d43c-00ef-4b95-b0d1-a20c22ba6921: !Template + ee64e45c-4405-45bb-b7c2-e37ee785fd7b: !Template answer_choices: A ||| B - id: fec5d43c-00ef-4b95-b0d1-a20c22ba6921 + id: ee64e45c-4405-45bb-b7c2-e37ee785fd7b jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} diff --git a/promptsource/templates/blimp/wh_vs_that_with_gap/templates.yaml b/promptsource/templates/blimp/wh_vs_that_with_gap/templates.yaml index b273160b2..ad2fe2b40 100644 --- a/promptsource/templates/blimp/wh_vs_that_with_gap/templates.yaml +++ b/promptsource/templates/blimp/wh_vs_that_with_gap/templates.yaml @@ -1,78 +1,53 @@ dataset: blimp subset: wh_vs_that_with_gap templates: - 06dbc186-11d3-4eec-bb9e-b229dbb87b20: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 06dbc186-11d3-4eec-bb9e-b229dbb87b20 - jinja: '" ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: null_prompt - reference: '' - 180ff240-8910-45b6-9992-7987729ec1df: !Template - answer_choices: A ||| B - id: 180ff240-8910-45b6-9992-7987729ec1df + 36f1f042-b559-4787-8d68-0aa4e5817fee: !Template + answer_choices: Yes ||| No + id: 36f1f042-b559-4787-8d68-0aa4e5817fee jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + {% set shuffled_order_options = [0, 1] | random %} - {% if shuffled_order == 0 %} + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" - Sentence A: {{ sentence_good }} - - Sentence B: {{ sentence_bad }} - - {% else %} - - Sentence A: {{ sentence_bad }} - - Sentence B: {{ sentence_good }} - - {% endif %} - - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} - - {% endif %}' + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy - original_task: true - name: better_B_A + original_task: false + name: single_sentence_good_yes_no reference: '' - 1b583b7d-5c3f-47dc-bda4-615a73ae111c: !Template - answer_choices: A ||| B - id: 1b583b7d-5c3f-47dc-bda4-615a73ae111c + 72645df4-fd34-4dcc-8bde-a51c1e9c1935: !Template + answer_choices: 1 ||| 2 + id: 72645df4-fd34-4dcc-8bde-a51c1e9c1935 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} - Sentence A: {{ sentence_good }} + 1: {{ sentence_good }} - Sentence B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - Sentence A: {{ sentence_bad }} + 1: {{ sentence_bad }} - Sentence B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + ||| + + {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -86,26 +61,29 @@ templates: metrics: - Accuracy original_task: true - name: better_A_B + name: grammatical_between_1_2 reference: '' - 48d300fd-82d7-409d-918a-e5bd39bf3c52: !Template - answer_choices: 1 ||| 2 - id: 48d300fd-82d7-409d-918a-e5bd39bf3c52 + 8610967f-a5fd-463e-a9d7-a5c9811f5784: !Template + answer_choices: A ||| B + id: 8610967f-a5fd-463e-a9d7-a5c9811f5784 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -125,42 +103,49 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 + name: grammatical_which_one_A_B reference: '' - 5ef7db69-d3a6-4279-864a-4941c7585954: !Template + 8fa1171d-d5ec-479a-bd24-eb24150bdd4c: !Template answer_choices: Yes ||| No - id: 5ef7db69-d3a6-4279-864a-4941c7585954 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + id: 8fa1171d-d5ec-479a-bd24-eb24150bdd4c + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_good_yes_no + name: single_sentence_bad_yes_no reference: '' - 6354bb80-ac81-40e4-838b-8619040863ac: !Template - answer_choices: 1 ||| 2 - id: 6354bb80-ac81-40e4-838b-8619040863ac + 93e1036a-f0f8-4fe0-804a-a3ba453879fb: !Template + answer_choices: A ||| B + id: 93e1036a-f0f8-4fe0-804a-a3ba453879fb jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -180,48 +165,33 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 - reference: '' - 64fd2d61-2190-4ad1-a891-b21cea26a62e: !Template - answer_choices: Yes ||| No - id: 64fd2d61-2190-4ad1-a891-b21cea26a62e - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no + name: grammatical_between_A_B reference: '' - 8a0f9f77-8cc7-4f43-bb71-ad1a7510f745: !Template + a4b730b4-dc8b-42c9-bb94-9bd679c7e326: !Template answer_choices: A ||| B - id: 8a0f9f77-8cc7-4f43-bb71-ad1a7510f745 + id: a4b730b4-dc8b-42c9-bb94-9bd679c7e326 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + Sentence A: {{ sentence_good }} - B: {{ sentence_bad }} + Sentence B: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + Sentence A: {{ sentence_bad }} - B: {{ sentence_good }} + Sentence B: {{ sentence_good }} {% endif %} - ||| - - {% if shuffled_order == 0 %} + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -235,42 +205,40 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: better_A_B reference: '' - 9f6df466-cf62-4cdb-9d6a-973e2e1192d6: !Template - answer_choices: Yes ||| No - id: 9f6df466-cf62-4cdb-9d6a-973e2e1192d6 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + ab520b24-0b0c-4719-b2ac-b3e89d9357ae: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: ab520b24-0b0c-4719-b2ac-b3e89d9357ae + jinja: ' ||| {{ sentence_good }}' metadata: !TemplateMetadata - choices_in_prompt: true + choices_in_prompt: false metrics: - Accuracy - original_task: false - name: single_sentence_good_no_yes + original_task: true + name: null_prompt reference: '' - a5b3813b-5adf-4a25-a632-700aa4b692cf: !Template - answer_choices: A ||| B - id: a5b3813b-5adf-4a25-a632-700aa4b692cf + e7adcae8-837a-40eb-a97e-e4fb5b8f596c: !Template + answer_choices: 1 ||| 2 + id: e7adcae8-837a-40eb-a97e-e4fb5b8f596c jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -290,32 +258,5 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_A_B - reference: '' - b86d7436-eff6-4205-8b1b-c634ba749f65: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: b86d7436-eff6-4205-8b1b-c634ba749f65 - jinja: ' ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: true_null_prompt - reference: '' - fe36c816-722d-4652-9235-04e9101f42f6: !Template - answer_choices: Yes ||| No - id: fe36c816-722d-4652-9235-04e9101f42f6 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes + name: grammatical_which_one_1_2 reference: '' diff --git a/promptsource/templates/blimp/wh_vs_that_with_gap_long_distance/templates.yaml b/promptsource/templates/blimp/wh_vs_that_with_gap_long_distance/templates.yaml index 712eae388..909435038 100644 --- a/promptsource/templates/blimp/wh_vs_that_with_gap_long_distance/templates.yaml +++ b/promptsource/templates/blimp/wh_vs_that_with_gap_long_distance/templates.yaml @@ -1,78 +1,27 @@ dataset: blimp subset: wh_vs_that_with_gap_long_distance templates: - 05d153b5-35ea-48a9-bf49-2469d840767b: !Template + 070af6c5-71af-481a-aded-bd51f24ee3f8: !Template answer_choices: A ||| B - id: 05d153b5-35ea-48a9-bf49-2469d840767b + id: 070af6c5-71af-481a-aded-bd51f24ee3f8 jinja: '{% set shuffled_order = [0, 1] | random %} - Read the following two sentences. - - {% if shuffled_order == 0 %} - - Sentence A: {{ sentence_good }} - - Sentence B: {{ sentence_bad }} - - {% else %} - - Sentence A: {{ sentence_bad }} - - Sentence B: {{ sentence_good }} - - {% endif %} - - Which one is a better sentence of English, B or A? ||| {% if shuffled_order - == 0 %} - - {{ answer_choices[0] }} - - {% else %} - - {{ answer_choices[1] }} - - {% endif %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: true - name: better_B_A - reference: '' - 1a6b7b9b-9b68-48a3-8192-23cc92d0f6a3: !Template - answer_choices: Yes ||| No - id: 1a6b7b9b-9b68-48a3-8192-23cc92d0f6a3 - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a good sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_good_yes_no - reference: '' - 253cc4c1-bfad-49e7-b9cb-fe9b1e82cbb3: !Template - answer_choices: 1 ||| 2 - id: 253cc4c1-bfad-49e7-b9cb-fe9b1e82cbb3 - jinja: '{% set shuffled_order = [0, 1] | random %} + {% set shuffled_order_options = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer 1 or 2. + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} - 1: {{ sentence_good }} + A: {{ sentence_good }} - 2: {{ sentence_bad }} + B: {{ sentence_bad }} {% else %} - 1: {{ sentence_bad }} + A: {{ sentence_bad }} - 2: {{ sentence_good }} + B: {{ sentence_good }} {% endif %} @@ -92,45 +41,15 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_1_2 - reference: '' - 274111ea-7de1-4e69-9298-7c21760d311b: !Template - answer_choices: Yes ||| No - id: 274111ea-7de1-4e69-9298-7c21760d311b - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_no_yes - reference: '' - 38988ff7-1854-4887-9b02-059bcc36199a: !Template - answer_choices: Yes ||| No - id: 38988ff7-1854-4887-9b02-059bcc36199a - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" - - Is this sentence a bad sentence of English, Yes or No? ||| {% if shuffled_order - == 0 %} {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif - %}' - metadata: !TemplateMetadata - choices_in_prompt: true - metrics: - - Accuracy - original_task: false - name: single_sentence_bad_yes_no + name: grammatical_between_A_B reference: '' - 4200922a-2bc2-4992-8408-2de5fdfcf435: !Template + 0f27d869-9944-4d07-a461-709f991294f7: !Template answer_choices: A ||| B - id: 4200922a-2bc2-4992-8408-2de5fdfcf435 + id: 0f27d869-9944-4d07-a461-709f991294f7 jinja: '{% set shuffled_order = [0, 1] | random %} + {% set shuffled_order_options = [0, 1] | random %} + Read the following two sentences. {% if shuffled_order == 0 %} @@ -147,8 +66,8 @@ templates: {% endif %} - Which one is a better sentence of English, A or B? ||| {% if shuffled_order - == 0 %} + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} {{ answer_choices[0] }} @@ -164,23 +83,15 @@ templates: original_task: true name: better_A_B reference: '' - 53101768-0016-4cfb-80a7-81581fcf7adc: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 53101768-0016-4cfb-80a7-81581fcf7adc - jinja: '" ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: null_prompt - reference: '' - 58644c43-ed51-4f3c-9f96-bea0bfe2892c: !Template + 16a913cc-56de-4785-acb1-e34c1f27c309: !Template answer_choices: A ||| B - id: 58644c43-ed51-4f3c-9f96-bea0bfe2892c + id: 16a913cc-56de-4785-acb1-e34c1f27c309 jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. {% if shuffled_order == 0 %} @@ -214,12 +125,46 @@ templates: original_task: true name: grammatical_which_one_A_B reference: '' - 5ea2aede-1e64-434d-a019-d9f969ca38ed: !Template + 4e312418-971d-405f-a25b-fd5b2f6c7444: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 4e312418-971d-405f-a25b-fd5b2f6c7444 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + 718c9c51-ee70-46bf-8663-0dfdcba56cae: !Template + answer_choices: Yes ||| No + id: 718c9c51-ee70-46bf-8663-0dfdcba56cae + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 87a19169-f4df-4c72-a1cd-2b394a7fffda: !Template answer_choices: 1 ||| 2 - id: 5ea2aede-1e64-434d-a019-d9f969ca38ed + id: 87a19169-f4df-4c72-a1cd-2b394a7fffda jinja: '{% set shuffled_order = [0, 1] | random %} - Which one of the following sentences is grammatical? Please answer 1 or 2. + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} @@ -251,53 +196,49 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_which_one_1_2 - reference: '' - 879299dd-fc1f-41cc-93b1-df2a7a141f8a: !Template - answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' - id: 879299dd-fc1f-41cc-93b1-df2a7a141f8a - jinja: ' ||| {{ sentence_good }}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - Accuracy - original_task: true - name: true_null_prompt + name: grammatical_between_1_2 reference: '' - c6f3615b-a641-4d8d-b104-d11a31b7da7e: !Template + 8878b790-491e-47da-aa7b-b216ec80223a: !Template answer_choices: Yes ||| No - id: c6f3615b-a641-4d8d-b104-d11a31b7da7e - jinja: '"{% set shuffled_order = [0, 1] | random %}{% if shuffled_order == 0 %}{{ - sentence_good }}{% else %}{{ sentence_bad }}{% endif %}" + id: 8878b790-491e-47da-aa7b-b216ec80223a + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} - Is this sentence a good sentence of English, No or Yes? ||| {% if shuffled_order - == 0 %} {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif - %}' + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' metadata: !TemplateMetadata choices_in_prompt: true metrics: - Accuracy original_task: false - name: single_sentence_good_no_yes + name: single_sentence_bad_yes_no reference: '' - ebc93391-c378-471e-b821-2e944a555018: !Template - answer_choices: A ||| B - id: ebc93391-c378-471e-b821-2e944a555018 + fea9a9cc-dc36-48de-92a4-e81bc3dd7220: !Template + answer_choices: 1 ||| 2 + id: fea9a9cc-dc36-48de-92a4-e81bc3dd7220 jinja: '{% set shuffled_order = [0, 1] | random %} - Between the two sentences, which one is grammatical? Please answer A or B. + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. {% if shuffled_order == 0 %} - A: {{ sentence_good }} + 1: {{ sentence_good }} - B: {{ sentence_bad }} + 2: {{ sentence_bad }} {% else %} - A: {{ sentence_bad }} + 1: {{ sentence_bad }} - B: {{ sentence_good }} + 2: {{ sentence_good }} {% endif %} @@ -317,5 +258,5 @@ templates: metrics: - Accuracy original_task: true - name: grammatical_between_A_B + name: grammatical_which_one_1_2 reference: ''