diff --git a/promptsource/templates/blimp/adjunct_island/templates.yaml b/promptsource/templates/blimp/adjunct_island/templates.yaml new file mode 100644 index 000000000..7181abf50 --- /dev/null +++ b/promptsource/templates/blimp/adjunct_island/templates.yaml @@ -0,0 +1,262 @@ +dataset: blimp +subset: adjunct_island +templates: + 6e71cd4b-882c-4458-9cd4-fd83f849ad09: !Template + answer_choices: 1 ||| 2 + id: 6e71cd4b-882c-4458-9cd4-fd83f849ad09 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_1_2 + reference: '' + 7ed733fd-524c-4f00-800f-77a7a862ab4a: !Template + answer_choices: 1 ||| 2 + id: 7ed733fd-524c-4f00-800f-77a7a862ab4a + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_1_2 + reference: '' + 8f354d74-5ffb-4cbc-b119-b79f2db00df3: !Template + answer_choices: Yes ||| No + id: 8f354d74-5ffb-4cbc-b119-b79f2db00df3 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 967ca4e2-162f-4365-a1f5-2c87c0e874ce: !Template + answer_choices: A ||| B + id: 967ca4e2-162f-4365-a1f5-2c87c0e874ce + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_A_B + reference: '' + a1997343-0a20-4f8c-8939-a512f0f68485: !Template + answer_choices: Yes ||| No + id: a1997343-0a20-4f8c-8939-a512f0f68485 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + c45d5bac-968e-4e39-96b7-d18a5ddb225c: !Template + answer_choices: A ||| B + id: c45d5bac-968e-4e39-96b7-d18a5ddb225c + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: better_A_B + reference: '' + d11a37d7-7c42-4ee0-96c5-ffd24ad7073e: !Template + answer_choices: A ||| B + id: d11a37d7-7c42-4ee0-96c5-ffd24ad7073e + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_A_B + reference: '' + f6cfe60f-a0c5-4ab8-ae7d-1c08d3cdaa2b: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: f6cfe60f-a0c5-4ab8-ae7d-1c08d3cdaa2b + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' diff --git a/promptsource/templates/blimp/anaphor_gender_agreement/templates.yaml b/promptsource/templates/blimp/anaphor_gender_agreement/templates.yaml new file mode 100644 index 000000000..73a8bd566 --- /dev/null +++ b/promptsource/templates/blimp/anaphor_gender_agreement/templates.yaml @@ -0,0 +1,262 @@ +dataset: blimp +subset: anaphor_gender_agreement +templates: + 07c38f60-08de-4ab9-9d85-10a6dc19d604: !Template + answer_choices: Yes ||| No + id: 07c38f60-08de-4ab9-9d85-10a6dc19d604 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 0bd4cee1-1ad7-491e-a035-da04db56a34a: !Template + answer_choices: Yes ||| No + id: 0bd4cee1-1ad7-491e-a035-da04db56a34a + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 4fc6563f-e07e-414c-836b-42ac0ad1bda1: !Template + answer_choices: 1 ||| 2 + id: 4fc6563f-e07e-414c-836b-42ac0ad1bda1 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_1_2 + reference: '' + 715c8035-7ea5-4083-9be3-d58da070a14f: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 715c8035-7ea5-4083-9be3-d58da070a14f + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + 77f3fbbd-88d3-432d-81fa-ed6efe7ed1ba: !Template + answer_choices: 1 ||| 2 + id: 77f3fbbd-88d3-432d-81fa-ed6efe7ed1ba + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_1_2 + reference: '' + d78bee9a-0745-4da9-a958-4a169a9c1da2: !Template + answer_choices: A ||| B + id: d78bee9a-0745-4da9-a958-4a169a9c1da2 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_A_B + reference: '' + f83533a9-239e-4534-90e6-51e6f2fcd581: !Template + answer_choices: A ||| B + id: f83533a9-239e-4534-90e6-51e6f2fcd581 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_A_B + reference: '' + fa337eec-293f-49ef-a6ab-19657333eef8: !Template + answer_choices: A ||| B + id: fa337eec-293f-49ef-a6ab-19657333eef8 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: better_A_B + reference: '' diff --git a/promptsource/templates/blimp/anaphor_number_agreement/templates.yaml b/promptsource/templates/blimp/anaphor_number_agreement/templates.yaml new file mode 100644 index 000000000..177f5739f --- /dev/null +++ b/promptsource/templates/blimp/anaphor_number_agreement/templates.yaml @@ -0,0 +1,262 @@ +dataset: blimp +subset: anaphor_number_agreement +templates: + 12e8c8b3-23b4-4134-87aa-35c16dcf1b28: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 12e8c8b3-23b4-4134-87aa-35c16dcf1b28 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + 37795465-6f54-4b97-8801-33fe6963d204: !Template + answer_choices: Yes ||| No + id: 37795465-6f54-4b97-8801-33fe6963d204 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 4b3a2b2c-945e-43a9-b1ed-f80e7a66989c: !Template + answer_choices: A ||| B + id: 4b3a2b2c-945e-43a9-b1ed-f80e7a66989c + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: better_A_B + reference: '' + 530a3b44-1c87-450e-8964-b349443ceb0c: !Template + answer_choices: 1 ||| 2 + id: 530a3b44-1c87-450e-8964-b349443ceb0c + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_1_2 + reference: '' + 80ebb026-8d6d-45c6-aad9-40d21601576e: !Template + answer_choices: 1 ||| 2 + id: 80ebb026-8d6d-45c6-aad9-40d21601576e + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_1_2 + reference: '' + 813e7a45-e5d1-4a68-9af6-91c6086c125c: !Template + answer_choices: A ||| B + id: 813e7a45-e5d1-4a68-9af6-91c6086c125c + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_A_B + reference: '' + a4da3db8-8075-4e34-bc38-53a0ac5dde9d: !Template + answer_choices: A ||| B + id: a4da3db8-8075-4e34-bc38-53a0ac5dde9d + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_A_B + reference: '' + e96b76bb-923c-4d9d-aa29-2a6ce4f5eb95: !Template + answer_choices: Yes ||| No + id: e96b76bb-923c-4d9d-aa29-2a6ce4f5eb95 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' diff --git a/promptsource/templates/blimp/animate_subject_passive/templates.yaml b/promptsource/templates/blimp/animate_subject_passive/templates.yaml new file mode 100644 index 000000000..16f9e8529 --- /dev/null +++ b/promptsource/templates/blimp/animate_subject_passive/templates.yaml @@ -0,0 +1,262 @@ +dataset: blimp +subset: animate_subject_passive +templates: + 2bce041a-2ef3-402f-9492-2b203e947897: !Template + answer_choices: Yes ||| No + id: 2bce041a-2ef3-402f-9492-2b203e947897 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 4e3231f6-f507-4f84-9017-2bf9852ad25e: !Template + answer_choices: A ||| B + id: 4e3231f6-f507-4f84-9017-2bf9852ad25e + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_A_B + reference: '' + 58ec546c-51f6-461b-9e3a-4eb816be86de: !Template + answer_choices: Yes ||| No + id: 58ec546c-51f6-461b-9e3a-4eb816be86de + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 61f671f5-9308-4627-b560-7c33587ae33b: !Template + answer_choices: 1 ||| 2 + id: 61f671f5-9308-4627-b560-7c33587ae33b + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_1_2 + reference: '' + 76fe5d87-47e1-40f2-896e-8ea3be79a613: !Template + answer_choices: 1 ||| 2 + id: 76fe5d87-47e1-40f2-896e-8ea3be79a613 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_1_2 + reference: '' + 7952fd9e-d112-4bba-92fc-24481c11f6a2: !Template + answer_choices: A ||| B + id: 7952fd9e-d112-4bba-92fc-24481c11f6a2 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_A_B + reference: '' + a745579b-fb13-456b-a14d-6a23e3d875f9: !Template + answer_choices: A ||| B + id: a745579b-fb13-456b-a14d-6a23e3d875f9 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: better_A_B + reference: '' + b36f421a-33d6-4e50-927e-01d5f75d8461: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: b36f421a-33d6-4e50-927e-01d5f75d8461 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' diff --git a/promptsource/templates/blimp/animate_subject_trans/templates.yaml b/promptsource/templates/blimp/animate_subject_trans/templates.yaml new file mode 100644 index 000000000..477cb7b90 --- /dev/null +++ b/promptsource/templates/blimp/animate_subject_trans/templates.yaml @@ -0,0 +1,262 @@ +dataset: blimp +subset: animate_subject_trans +templates: + 0e668f2a-2d52-4619-9225-8a934defb775: !Template + answer_choices: Yes ||| No + id: 0e668f2a-2d52-4619-9225-8a934defb775 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 20018611-aea5-4866-b3b7-74ec0b68920e: !Template + answer_choices: A ||| B + id: 20018611-aea5-4866-b3b7-74ec0b68920e + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: better_A_B + reference: '' + 3dda3756-4efa-4f0a-a772-1a41262e5476: !Template + answer_choices: A ||| B + id: 3dda3756-4efa-4f0a-a772-1a41262e5476 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_A_B + reference: '' + 9e0965c1-c2bd-4dc1-bf8c-f310d73f5758: !Template + answer_choices: 1 ||| 2 + id: 9e0965c1-c2bd-4dc1-bf8c-f310d73f5758 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_1_2 + reference: '' + dc62e655-3192-49f7-ace1-7b14732fab8c: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: dc62e655-3192-49f7-ace1-7b14732fab8c + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + e91b1182-854e-4473-b9f2-8f68c4d72c8c: !Template + answer_choices: A ||| B + id: e91b1182-854e-4473-b9f2-8f68c4d72c8c + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_A_B + reference: '' + ebe63f74-9c6f-4a29-9fb1-2567c5354609: !Template + answer_choices: 1 ||| 2 + id: ebe63f74-9c6f-4a29-9fb1-2567c5354609 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_1_2 + reference: '' + f7936b93-8a2e-4ae6-b85f-a494ed49a1b3: !Template + answer_choices: Yes ||| No + id: f7936b93-8a2e-4ae6-b85f-a494ed49a1b3 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' diff --git a/promptsource/templates/blimp/causative/templates.yaml b/promptsource/templates/blimp/causative/templates.yaml new file mode 100644 index 000000000..9994ae8db --- /dev/null +++ b/promptsource/templates/blimp/causative/templates.yaml @@ -0,0 +1,262 @@ +dataset: blimp +subset: causative +templates: + 0107b558-5748-4807-9238-9dc921b36220: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 0107b558-5748-4807-9238-9dc921b36220 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + 0a3230e8-2cce-4c41-aaef-eee10cb200e3: !Template + answer_choices: Yes ||| No + id: 0a3230e8-2cce-4c41-aaef-eee10cb200e3 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 2a1d7c07-be9e-4980-81e3-f36c4453b834: !Template + answer_choices: 1 ||| 2 + id: 2a1d7c07-be9e-4980-81e3-f36c4453b834 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_1_2 + reference: '' + 596e688d-2267-47e1-a427-18e256798411: !Template + answer_choices: 1 ||| 2 + id: 596e688d-2267-47e1-a427-18e256798411 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_1_2 + reference: '' + 6b8a30a4-d070-4029-b93d-8f18c9247140: !Template + answer_choices: A ||| B + id: 6b8a30a4-d070-4029-b93d-8f18c9247140 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_A_B + reference: '' + b3bd8baf-0fcb-4d47-91fd-72bdbce631de: !Template + answer_choices: Yes ||| No + id: b3bd8baf-0fcb-4d47-91fd-72bdbce631de + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + c7852c5e-08be-4672-b534-b560099db2af: !Template + answer_choices: A ||| B + id: c7852c5e-08be-4672-b534-b560099db2af + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_A_B + reference: '' + f53e9bec-4fca-44cb-aa76-bf245d3f9fb7: !Template + answer_choices: A ||| B + id: f53e9bec-4fca-44cb-aa76-bf245d3f9fb7 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: better_A_B + reference: '' diff --git a/promptsource/templates/blimp/complex_NP_island/templates.yaml b/promptsource/templates/blimp/complex_NP_island/templates.yaml new file mode 100644 index 000000000..7fdc8291a --- /dev/null +++ b/promptsource/templates/blimp/complex_NP_island/templates.yaml @@ -0,0 +1,262 @@ +dataset: blimp +subset: complex_NP_island +templates: + 37f9405d-9482-4691-8853-945a22210b9e: !Template + answer_choices: A ||| B + id: 37f9405d-9482-4691-8853-945a22210b9e + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_A_B + reference: '' + 3cd258c4-b303-4884-9db4-4ef3d89a1ef2: !Template + answer_choices: Yes ||| No + id: 3cd258c4-b303-4884-9db4-4ef3d89a1ef2 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 3e4570a1-bf6c-43e1-9566-6054587f9203: !Template + answer_choices: A ||| B + id: 3e4570a1-bf6c-43e1-9566-6054587f9203 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_A_B + reference: '' + 6d115be5-5911-48ef-ae43-88b92defca66: !Template + answer_choices: 1 ||| 2 + id: 6d115be5-5911-48ef-ae43-88b92defca66 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_1_2 + reference: '' + 6ed7e579-3927-4646-890d-1c0ebc2ffc60: !Template + answer_choices: Yes ||| No + id: 6ed7e579-3927-4646-890d-1c0ebc2ffc60 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 9c282484-a3da-47c4-afcd-6363d744f558: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 9c282484-a3da-47c4-afcd-6363d744f558 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + b3f7db9b-5363-4076-a966-ef6b17d90991: !Template + answer_choices: A ||| B + id: b3f7db9b-5363-4076-a966-ef6b17d90991 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: better_A_B + reference: '' + e12a2257-9641-47a4-bcaf-d674e35b77e8: !Template + answer_choices: 1 ||| 2 + id: e12a2257-9641-47a4-bcaf-d674e35b77e8 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_1_2 + reference: '' diff --git a/promptsource/templates/blimp/coordinate_structure_constraint_complex_left_branch/templates.yaml b/promptsource/templates/blimp/coordinate_structure_constraint_complex_left_branch/templates.yaml new file mode 100644 index 000000000..c44a45b3d --- /dev/null +++ b/promptsource/templates/blimp/coordinate_structure_constraint_complex_left_branch/templates.yaml @@ -0,0 +1,262 @@ +dataset: blimp +subset: coordinate_structure_constraint_complex_left_branch +templates: + 0e93a8db-4f93-4ac3-9502-6a8132c2a647: !Template + answer_choices: Yes ||| No + id: 0e93a8db-4f93-4ac3-9502-6a8132c2a647 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 1630a692-c39a-447f-a672-9c441fffd3e3: !Template + answer_choices: 1 ||| 2 + id: 1630a692-c39a-447f-a672-9c441fffd3e3 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_1_2 + reference: '' + 29b55f60-5b70-4dc2-93fd-e03ce2e0b8cc: !Template + answer_choices: A ||| B + id: 29b55f60-5b70-4dc2-93fd-e03ce2e0b8cc + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: better_A_B + reference: '' + 3e2be3c5-e21b-4cbd-9bdb-a951f698de92: !Template + answer_choices: 1 ||| 2 + id: 3e2be3c5-e21b-4cbd-9bdb-a951f698de92 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_1_2 + reference: '' + 4006ea8e-e9ec-4605-b230-0e9a2415eb66: !Template + answer_choices: A ||| B + id: 4006ea8e-e9ec-4605-b230-0e9a2415eb66 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_A_B + reference: '' + 89ca822d-5cf7-4362-aadc-df69544ef4e3: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 89ca822d-5cf7-4362-aadc-df69544ef4e3 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + 90da64af-ad98-4229-a2d7-b04191f55b1c: !Template + answer_choices: Yes ||| No + id: 90da64af-ad98-4229-a2d7-b04191f55b1c + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + fe0a2557-7de0-4b98-b2cb-28aa03d33e44: !Template + answer_choices: A ||| B + id: fe0a2557-7de0-4b98-b2cb-28aa03d33e44 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_A_B + reference: '' diff --git a/promptsource/templates/blimp/coordinate_structure_constraint_object_extraction/templates.yaml b/promptsource/templates/blimp/coordinate_structure_constraint_object_extraction/templates.yaml new file mode 100644 index 000000000..c5917839c --- /dev/null +++ b/promptsource/templates/blimp/coordinate_structure_constraint_object_extraction/templates.yaml @@ -0,0 +1,262 @@ +dataset: blimp +subset: coordinate_structure_constraint_object_extraction +templates: + 048bbd16-16bd-4aa3-a0cb-f65d38eb5193: !Template + answer_choices: A ||| B + id: 048bbd16-16bd-4aa3-a0cb-f65d38eb5193 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: better_A_B + reference: '' + 351916b8-be3d-45d0-b7ff-1e2c8f317449: !Template + answer_choices: Yes ||| No + id: 351916b8-be3d-45d0-b7ff-1e2c8f317449 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 47d445a6-5a8e-428e-8520-b0084d46e2ca: !Template + answer_choices: A ||| B + id: 47d445a6-5a8e-428e-8520-b0084d46e2ca + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_A_B + reference: '' + 73858b20-2880-4fd6-9c44-abd3c0af91cd: !Template + answer_choices: 1 ||| 2 + id: 73858b20-2880-4fd6-9c44-abd3c0af91cd + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_1_2 + reference: '' + a1e4dafa-66b6-49b1-806e-a53ab60aeb73: !Template + answer_choices: 1 ||| 2 + id: a1e4dafa-66b6-49b1-806e-a53ab60aeb73 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_1_2 + reference: '' + ae2b3124-dd9d-4b78-a2da-f9168345fd99: !Template + answer_choices: A ||| B + id: ae2b3124-dd9d-4b78-a2da-f9168345fd99 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_A_B + reference: '' + b4a33a5f-cc31-4366-a84d-e97b8129a144: !Template + answer_choices: Yes ||| No + id: b4a33a5f-cc31-4366-a84d-e97b8129a144 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + c2c9b04a-5158-4d0a-b170-17c0cf8d52de: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: c2c9b04a-5158-4d0a-b170-17c0cf8d52de + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' diff --git a/promptsource/templates/blimp/determiner_noun_agreement_1/templates.yaml b/promptsource/templates/blimp/determiner_noun_agreement_1/templates.yaml new file mode 100644 index 000000000..47020325b --- /dev/null +++ b/promptsource/templates/blimp/determiner_noun_agreement_1/templates.yaml @@ -0,0 +1,262 @@ +dataset: blimp +subset: determiner_noun_agreement_1 +templates: + 03d16d6f-bfdd-4b84-919f-e6c6b81f3c88: !Template + answer_choices: 1 ||| 2 + id: 03d16d6f-bfdd-4b84-919f-e6c6b81f3c88 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_1_2 + reference: '' + 23ebe991-1aeb-4873-b9d5-c5235cf886c6: !Template + answer_choices: A ||| B + id: 23ebe991-1aeb-4873-b9d5-c5235cf886c6 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_A_B + reference: '' + 24f8db17-2c78-44a2-b8b9-db2237818623: !Template + answer_choices: Yes ||| No + id: 24f8db17-2c78-44a2-b8b9-db2237818623 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 341495b5-cf05-4c77-9678-56c5039ca446: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 341495b5-cf05-4c77-9678-56c5039ca446 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + 4d771189-3657-43f0-bf82-984ad14ebfa8: !Template + answer_choices: Yes ||| No + id: 4d771189-3657-43f0-bf82-984ad14ebfa8 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 89eed9d7-5e70-4948-b39b-4bd03b2ffc5f: !Template + answer_choices: 1 ||| 2 + id: 89eed9d7-5e70-4948-b39b-4bd03b2ffc5f + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_1_2 + reference: '' + 8a9e9fae-a83d-456f-8b5b-df688ee1bad2: !Template + answer_choices: A ||| B + id: 8a9e9fae-a83d-456f-8b5b-df688ee1bad2 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: better_A_B + reference: '' + f8850ec9-845d-4271-b691-71068a0f45f9: !Template + answer_choices: A ||| B + id: f8850ec9-845d-4271-b691-71068a0f45f9 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_A_B + reference: '' diff --git a/promptsource/templates/blimp/determiner_noun_agreement_2/templates.yaml b/promptsource/templates/blimp/determiner_noun_agreement_2/templates.yaml new file mode 100644 index 000000000..5b3a03415 --- /dev/null +++ b/promptsource/templates/blimp/determiner_noun_agreement_2/templates.yaml @@ -0,0 +1,262 @@ +dataset: blimp +subset: determiner_noun_agreement_2 +templates: + 1ff49155-9e74-463f-9bea-5ea0f1c4c5cc: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 1ff49155-9e74-463f-9bea-5ea0f1c4c5cc + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + 49f63f3b-5f6a-4663-a4f3-6e419e2c8e46: !Template + answer_choices: Yes ||| No + id: 49f63f3b-5f6a-4663-a4f3-6e419e2c8e46 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 4b2714f1-b970-4f26-9893-89a4b887260f: !Template + answer_choices: A ||| B + id: 4b2714f1-b970-4f26-9893-89a4b887260f + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_A_B + reference: '' + 6a002670-0148-485a-9278-e6951bd60409: !Template + answer_choices: A ||| B + id: 6a002670-0148-485a-9278-e6951bd60409 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: better_A_B + reference: '' + b2e18f6d-69c2-414c-b36f-01eadcd23f39: !Template + answer_choices: Yes ||| No + id: b2e18f6d-69c2-414c-b36f-01eadcd23f39 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + ca5ce2c1-4690-49d4-a18d-53431b9b2a1d: !Template + answer_choices: 1 ||| 2 + id: ca5ce2c1-4690-49d4-a18d-53431b9b2a1d + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_1_2 + reference: '' + e8c2e7c6-9197-44c8-a14c-4bac6fa7df44: !Template + answer_choices: 1 ||| 2 + id: e8c2e7c6-9197-44c8-a14c-4bac6fa7df44 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_1_2 + reference: '' + ee66157f-f99e-4431-bfdb-cf041ac21c52: !Template + answer_choices: A ||| B + id: ee66157f-f99e-4431-bfdb-cf041ac21c52 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_A_B + reference: '' diff --git a/promptsource/templates/blimp/determiner_noun_agreement_irregular_1/templates.yaml b/promptsource/templates/blimp/determiner_noun_agreement_irregular_1/templates.yaml new file mode 100644 index 000000000..67072388b --- /dev/null +++ b/promptsource/templates/blimp/determiner_noun_agreement_irregular_1/templates.yaml @@ -0,0 +1,262 @@ +dataset: blimp +subset: determiner_noun_agreement_irregular_1 +templates: + 049b2393-dd62-4ce4-bba4-ee0afc7c3ec4: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 049b2393-dd62-4ce4-bba4-ee0afc7c3ec4 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + 3774724e-fa89-427f-acfe-e176cf2459a4: !Template + answer_choices: 1 ||| 2 + id: 3774724e-fa89-427f-acfe-e176cf2459a4 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_1_2 + reference: '' + 41f2912c-1126-4940-91f1-3b2b837d8460: !Template + answer_choices: 1 ||| 2 + id: 41f2912c-1126-4940-91f1-3b2b837d8460 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_1_2 + reference: '' + 85cf8fa9-af9d-4ade-8113-0022cbb4a295: !Template + answer_choices: A ||| B + id: 85cf8fa9-af9d-4ade-8113-0022cbb4a295 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_A_B + reference: '' + a17cf3be-ae52-4cd1-a2a3-348bf74acfa7: !Template + answer_choices: A ||| B + id: a17cf3be-ae52-4cd1-a2a3-348bf74acfa7 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: better_A_B + reference: '' + a48cef96-2d21-4205-aaf8-e46396f3a872: !Template + answer_choices: Yes ||| No + id: a48cef96-2d21-4205-aaf8-e46396f3a872 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + cd6fd211-5af5-4d83-8282-5df007f05a6d: !Template + answer_choices: A ||| B + id: cd6fd211-5af5-4d83-8282-5df007f05a6d + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_A_B + reference: '' + e5ee12bd-b051-4487-8abd-1e0535b1f9cd: !Template + answer_choices: Yes ||| No + id: e5ee12bd-b051-4487-8abd-1e0535b1f9cd + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' diff --git a/promptsource/templates/blimp/determiner_noun_agreement_irregular_2/templates.yaml b/promptsource/templates/blimp/determiner_noun_agreement_irregular_2/templates.yaml new file mode 100644 index 000000000..ee2dbfc88 --- /dev/null +++ b/promptsource/templates/blimp/determiner_noun_agreement_irregular_2/templates.yaml @@ -0,0 +1,262 @@ +dataset: blimp +subset: determiner_noun_agreement_irregular_2 +templates: + 124e6432-02e3-4e07-9ada-ff24ccd71656: !Template + answer_choices: Yes ||| No + id: 124e6432-02e3-4e07-9ada-ff24ccd71656 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 8406d6c0-2383-4832-b0ff-48ca295944f2: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 8406d6c0-2383-4832-b0ff-48ca295944f2 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + 86eebca6-aac0-4949-9b00-76ad6fefdda0: !Template + answer_choices: Yes ||| No + id: 86eebca6-aac0-4949-9b00-76ad6fefdda0 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + ca5311f3-7066-4e65-8764-64319087ebfa: !Template + answer_choices: A ||| B + id: ca5311f3-7066-4e65-8764-64319087ebfa + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_A_B + reference: '' + d7f7688c-86d4-4a29-8c4e-e25f016db83b: !Template + answer_choices: A ||| B + id: d7f7688c-86d4-4a29-8c4e-e25f016db83b + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: better_A_B + reference: '' + e43b8098-ea0f-4786-af61-9bf3f744a7be: !Template + answer_choices: A ||| B + id: e43b8098-ea0f-4786-af61-9bf3f744a7be + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_A_B + reference: '' + fc0d9c16-020e-4bd8-91d9-7e56bd56146c: !Template + answer_choices: 1 ||| 2 + id: fc0d9c16-020e-4bd8-91d9-7e56bd56146c + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_1_2 + reference: '' + fdfdef44-308e-4b31-9477-0fefe55a20f5: !Template + answer_choices: 1 ||| 2 + id: fdfdef44-308e-4b31-9477-0fefe55a20f5 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_1_2 + reference: '' diff --git a/promptsource/templates/blimp/determiner_noun_agreement_with_adj_2/templates.yaml b/promptsource/templates/blimp/determiner_noun_agreement_with_adj_2/templates.yaml new file mode 100644 index 000000000..82c4ccc41 --- /dev/null +++ b/promptsource/templates/blimp/determiner_noun_agreement_with_adj_2/templates.yaml @@ -0,0 +1,262 @@ +dataset: blimp +subset: determiner_noun_agreement_with_adj_2 +templates: + 062fc639-5b95-445b-a161-cb3435c3917e: !Template + answer_choices: 1 ||| 2 + id: 062fc639-5b95-445b-a161-cb3435c3917e + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_1_2 + reference: '' + 12ebdf0e-4d2e-413a-b356-b20f26d697f9: !Template + answer_choices: A ||| B + id: 12ebdf0e-4d2e-413a-b356-b20f26d697f9 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_A_B + reference: '' + 1c886997-c2f6-49fa-8f45-94fe821576da: !Template + answer_choices: A ||| B + id: 1c886997-c2f6-49fa-8f45-94fe821576da + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_A_B + reference: '' + 207eac77-5fdc-4351-8d58-41a490cddafa: !Template + answer_choices: Yes ||| No + id: 207eac77-5fdc-4351-8d58-41a490cddafa + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 6cab34a8-4125-4e00-96c9-5adfa41b40cd: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 6cab34a8-4125-4e00-96c9-5adfa41b40cd + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + 9cea593b-b7cf-409e-8b4e-3fe1ee52a116: !Template + answer_choices: 1 ||| 2 + id: 9cea593b-b7cf-409e-8b4e-3fe1ee52a116 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_1_2 + reference: '' + 9e000ee9-c511-43ff-b3b8-0487681b60f5: !Template + answer_choices: Yes ||| No + id: 9e000ee9-c511-43ff-b3b8-0487681b60f5 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + d24e5794-5f01-4bae-bd5d-7a8c83ef33be: !Template + answer_choices: A ||| B + id: d24e5794-5f01-4bae-bd5d-7a8c83ef33be + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: better_A_B + reference: '' diff --git a/promptsource/templates/blimp/determiner_noun_agreement_with_adj_irregular_1/templates.yaml b/promptsource/templates/blimp/determiner_noun_agreement_with_adj_irregular_1/templates.yaml new file mode 100644 index 000000000..d6f0af25a --- /dev/null +++ b/promptsource/templates/blimp/determiner_noun_agreement_with_adj_irregular_1/templates.yaml @@ -0,0 +1,262 @@ +dataset: blimp +subset: determiner_noun_agreement_with_adj_irregular_1 +templates: + 2a96a2a0-b391-44aa-991f-a1dd3daa2c83: !Template + answer_choices: A ||| B + id: 2a96a2a0-b391-44aa-991f-a1dd3daa2c83 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: better_A_B + reference: '' + 2c819b61-7921-4827-a02c-c269cb857432: !Template + answer_choices: 1 ||| 2 + id: 2c819b61-7921-4827-a02c-c269cb857432 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_1_2 + reference: '' + 3c996321-603c-4d6f-a536-784916f9ca54: !Template + answer_choices: Yes ||| No + id: 3c996321-603c-4d6f-a536-784916f9ca54 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 55a34363-0c99-4f77-94b3-72055c9deff0: !Template + answer_choices: Yes ||| No + id: 55a34363-0c99-4f77-94b3-72055c9deff0 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 782fff30-473e-492f-9570-edacf7b52b06: !Template + answer_choices: 1 ||| 2 + id: 782fff30-473e-492f-9570-edacf7b52b06 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_1_2 + reference: '' + 8257567f-5280-4e6a-89ac-4aed01c1befb: !Template + answer_choices: A ||| B + id: 8257567f-5280-4e6a-89ac-4aed01c1befb + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_A_B + reference: '' + c8555faf-50e2-4785-ad8a-41bf325016a6: !Template + answer_choices: A ||| B + id: c8555faf-50e2-4785-ad8a-41bf325016a6 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_A_B + reference: '' + f2b79942-dd9f-40a6-8ef5-96e8b519b08f: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: f2b79942-dd9f-40a6-8ef5-96e8b519b08f + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' diff --git a/promptsource/templates/blimp/determiner_noun_agreement_with_adj_irregular_2/templates.yaml b/promptsource/templates/blimp/determiner_noun_agreement_with_adj_irregular_2/templates.yaml new file mode 100644 index 000000000..a417782e1 --- /dev/null +++ b/promptsource/templates/blimp/determiner_noun_agreement_with_adj_irregular_2/templates.yaml @@ -0,0 +1,262 @@ +dataset: blimp +subset: determiner_noun_agreement_with_adj_irregular_2 +templates: + 05972c48-b382-43ba-9afb-8664220284b4: !Template + answer_choices: A ||| B + id: 05972c48-b382-43ba-9afb-8664220284b4 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_A_B + reference: '' + 143373b3-b7c7-42f8-924f-b7e308218c4c: !Template + answer_choices: Yes ||| No + id: 143373b3-b7c7-42f8-924f-b7e308218c4c + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 30faf1bc-e1ba-492a-b07a-a8e6eb06d248: !Template + answer_choices: A ||| B + id: 30faf1bc-e1ba-492a-b07a-a8e6eb06d248 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: better_A_B + reference: '' + 509b6572-71f2-49a7-ab5b-bf1a3bb869a8: !Template + answer_choices: 1 ||| 2 + id: 509b6572-71f2-49a7-ab5b-bf1a3bb869a8 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_1_2 + reference: '' + 6630731c-8593-4471-a0d1-b3547a176bdb: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 6630731c-8593-4471-a0d1-b3547a176bdb + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + 8f65d37e-8e47-429b-b56d-c6762842663b: !Template + answer_choices: A ||| B + id: 8f65d37e-8e47-429b-b56d-c6762842663b + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_A_B + reference: '' + d7a8ee44-8bb6-4575-b1e0-b1b36461a0fa: !Template + answer_choices: Yes ||| No + id: d7a8ee44-8bb6-4575-b1e0-b1b36461a0fa + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + f9ae1c45-9f80-4c9b-88fb-6969c72cc5ed: !Template + answer_choices: 1 ||| 2 + id: f9ae1c45-9f80-4c9b-88fb-6969c72cc5ed + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_1_2 + reference: '' diff --git a/promptsource/templates/blimp/determiner_noun_agreement_with_adjective_1/templates.yaml b/promptsource/templates/blimp/determiner_noun_agreement_with_adjective_1/templates.yaml new file mode 100644 index 000000000..679e608c2 --- /dev/null +++ b/promptsource/templates/blimp/determiner_noun_agreement_with_adjective_1/templates.yaml @@ -0,0 +1,262 @@ +dataset: blimp +subset: determiner_noun_agreement_with_adjective_1 +templates: + 16fc0d72-7188-400a-9373-6612ed85dca4: !Template + answer_choices: A ||| B + id: 16fc0d72-7188-400a-9373-6612ed85dca4 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_A_B + reference: '' + 3403d371-ebcf-4212-b1ca-1d392f7e9907: !Template + answer_choices: 1 ||| 2 + id: 3403d371-ebcf-4212-b1ca-1d392f7e9907 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_1_2 + reference: '' + 3ee681df-24b4-43f8-a9de-39f5c8614d7d: !Template + answer_choices: A ||| B + id: 3ee681df-24b4-43f8-a9de-39f5c8614d7d + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: better_A_B + reference: '' + 48036274-c1c3-4e30-8fcc-bb159797cbbf: !Template + answer_choices: Yes ||| No + id: 48036274-c1c3-4e30-8fcc-bb159797cbbf + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 8261a763-d444-46a2-a223-891370d55245: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 8261a763-d444-46a2-a223-891370d55245 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + 87eeacd9-a39a-4dc3-9814-85410875666a: !Template + answer_choices: A ||| B + id: 87eeacd9-a39a-4dc3-9814-85410875666a + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_A_B + reference: '' + a0eba446-b21c-41ff-bbbc-e55c40bcebbf: !Template + answer_choices: Yes ||| No + id: a0eba446-b21c-41ff-bbbc-e55c40bcebbf + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + fd1cc83f-521c-4b3d-8b86-fe0ce9e4cbcf: !Template + answer_choices: 1 ||| 2 + id: fd1cc83f-521c-4b3d-8b86-fe0ce9e4cbcf + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_1_2 + reference: '' diff --git a/promptsource/templates/blimp/distractor_agreement_relational_noun/templates.yaml b/promptsource/templates/blimp/distractor_agreement_relational_noun/templates.yaml new file mode 100644 index 000000000..49b28e325 --- /dev/null +++ b/promptsource/templates/blimp/distractor_agreement_relational_noun/templates.yaml @@ -0,0 +1,262 @@ +dataset: blimp +subset: distractor_agreement_relational_noun +templates: + 04e6bf51-c73a-4089-986b-789729e3db62: !Template + answer_choices: A ||| B + id: 04e6bf51-c73a-4089-986b-789729e3db62 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_A_B + reference: '' + 37b91041-3225-4291-baa3-33a85d3c6071: !Template + answer_choices: 1 ||| 2 + id: 37b91041-3225-4291-baa3-33a85d3c6071 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_1_2 + reference: '' + 70527b1b-38e3-4e50-9142-3a6a4b068d35: !Template + answer_choices: Yes ||| No + id: 70527b1b-38e3-4e50-9142-3a6a4b068d35 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 736d891c-e000-4d0f-b2df-b343c501ad2f: !Template + answer_choices: A ||| B + id: 736d891c-e000-4d0f-b2df-b343c501ad2f + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: better_A_B + reference: '' + dfb67083-db29-415c-bdc8-6f25b97a1293: !Template + answer_choices: A ||| B + id: dfb67083-db29-415c-bdc8-6f25b97a1293 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_A_B + reference: '' + e0e9dadb-9638-4b87-a0f4-666e5c17b94e: !Template + answer_choices: Yes ||| No + id: e0e9dadb-9638-4b87-a0f4-666e5c17b94e + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + e7e10ac3-980a-4d0f-874e-72210e1f42cf: !Template + answer_choices: 1 ||| 2 + id: e7e10ac3-980a-4d0f-874e-72210e1f42cf + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_1_2 + reference: '' + edc1ffa1-ef01-418f-b5e7-14f2d5f0f801: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: edc1ffa1-ef01-418f-b5e7-14f2d5f0f801 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' diff --git a/promptsource/templates/blimp/distractor_agreement_relative_clause/templates.yaml b/promptsource/templates/blimp/distractor_agreement_relative_clause/templates.yaml new file mode 100644 index 000000000..ece833081 --- /dev/null +++ b/promptsource/templates/blimp/distractor_agreement_relative_clause/templates.yaml @@ -0,0 +1,262 @@ +dataset: blimp +subset: distractor_agreement_relative_clause +templates: + 031e9719-1408-4a21-8fdf-3bad71fbb2fc: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 031e9719-1408-4a21-8fdf-3bad71fbb2fc + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + 343cf095-39b4-4e4e-912b-84b116217991: !Template + answer_choices: 1 ||| 2 + id: 343cf095-39b4-4e4e-912b-84b116217991 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_1_2 + reference: '' + 38c20a09-df68-46e7-9f75-7e08474fb3d9: !Template + answer_choices: A ||| B + id: 38c20a09-df68-46e7-9f75-7e08474fb3d9 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_A_B + reference: '' + 6a81bb6e-2c7e-496e-80e0-fc49587f9334: !Template + answer_choices: 1 ||| 2 + id: 6a81bb6e-2c7e-496e-80e0-fc49587f9334 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_1_2 + reference: '' + 8256789f-0e85-4bd9-b019-98fcabdeaa77: !Template + answer_choices: Yes ||| No + id: 8256789f-0e85-4bd9-b019-98fcabdeaa77 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 8c8ee811-a4b0-4822-8c03-f53016f87554: !Template + answer_choices: Yes ||| No + id: 8c8ee811-a4b0-4822-8c03-f53016f87554 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 99f31214-16a8-4934-895d-30e0f3551426: !Template + answer_choices: A ||| B + id: 99f31214-16a8-4934-895d-30e0f3551426 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: better_A_B + reference: '' + ff14c20d-654c-4eca-b397-9f873d8df853: !Template + answer_choices: A ||| B + id: ff14c20d-654c-4eca-b397-9f873d8df853 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_A_B + reference: '' diff --git a/promptsource/templates/blimp/drop_argument/templates.yaml b/promptsource/templates/blimp/drop_argument/templates.yaml new file mode 100644 index 000000000..39bb80d3f --- /dev/null +++ b/promptsource/templates/blimp/drop_argument/templates.yaml @@ -0,0 +1,262 @@ +dataset: blimp +subset: drop_argument +templates: + 2d421675-b645-4f6c-b166-a3f79b16be9b: !Template + answer_choices: Yes ||| No + id: 2d421675-b645-4f6c-b166-a3f79b16be9b + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 4c353c08-d69b-48cd-8003-17f3e03da2f3: !Template + answer_choices: A ||| B + id: 4c353c08-d69b-48cd-8003-17f3e03da2f3 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_A_B + reference: '' + 53cc7b68-cf20-44f0-ba0a-c12289e82e95: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 53cc7b68-cf20-44f0-ba0a-c12289e82e95 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + 854e5efa-794f-48dd-a3fe-7027064af3ee: !Template + answer_choices: 1 ||| 2 + id: 854e5efa-794f-48dd-a3fe-7027064af3ee + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_1_2 + reference: '' + 9398d73d-00a0-48f2-b667-e2ea8ab743ec: !Template + answer_choices: Yes ||| No + id: 9398d73d-00a0-48f2-b667-e2ea8ab743ec + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 9f31c997-9c83-4ee4-a879-354047af5756: !Template + answer_choices: 1 ||| 2 + id: 9f31c997-9c83-4ee4-a879-354047af5756 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_1_2 + reference: '' + cb6ac991-e5a3-4325-beca-fb4ec479ce45: !Template + answer_choices: A ||| B + id: cb6ac991-e5a3-4325-beca-fb4ec479ce45 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: better_A_B + reference: '' + f2204727-5f8c-491f-aff8-317fb4d87e1d: !Template + answer_choices: A ||| B + id: f2204727-5f8c-491f-aff8-317fb4d87e1d + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_A_B + reference: '' diff --git a/promptsource/templates/blimp/ellipsis_n_bar_1/templates.yaml b/promptsource/templates/blimp/ellipsis_n_bar_1/templates.yaml new file mode 100644 index 000000000..e47ceaebc --- /dev/null +++ b/promptsource/templates/blimp/ellipsis_n_bar_1/templates.yaml @@ -0,0 +1,262 @@ +dataset: blimp +subset: ellipsis_n_bar_1 +templates: + 059b8cb5-6372-4986-9015-e26c43a1294f: !Template + answer_choices: Yes ||| No + id: 059b8cb5-6372-4986-9015-e26c43a1294f + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 11f3090a-0a82-40be-a007-78f492e4ae97: !Template + answer_choices: Yes ||| No + id: 11f3090a-0a82-40be-a007-78f492e4ae97 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 1afe2ffd-1348-49bf-a893-9558832c2276: !Template + answer_choices: A ||| B + id: 1afe2ffd-1348-49bf-a893-9558832c2276 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_A_B + reference: '' + 1d77964b-9969-4312-be69-9c628c2adaee: !Template + answer_choices: A ||| B + id: 1d77964b-9969-4312-be69-9c628c2adaee + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: better_A_B + reference: '' + 66090200-488a-4802-95ff-05c6aba62170: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 66090200-488a-4802-95ff-05c6aba62170 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + 999aa38b-d71f-4610-99c8-238bb7dde54a: !Template + answer_choices: 1 ||| 2 + id: 999aa38b-d71f-4610-99c8-238bb7dde54a + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_1_2 + reference: '' + d5fad176-7509-421a-b8ee-757976985f3d: !Template + answer_choices: 1 ||| 2 + id: d5fad176-7509-421a-b8ee-757976985f3d + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_1_2 + reference: '' + eab1c212-810a-4a09-94a3-d7bf35697910: !Template + answer_choices: A ||| B + id: eab1c212-810a-4a09-94a3-d7bf35697910 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_A_B + reference: '' diff --git a/promptsource/templates/blimp/ellipsis_n_bar_2/templates.yaml b/promptsource/templates/blimp/ellipsis_n_bar_2/templates.yaml new file mode 100644 index 000000000..7fd3f5f99 --- /dev/null +++ b/promptsource/templates/blimp/ellipsis_n_bar_2/templates.yaml @@ -0,0 +1,262 @@ +dataset: blimp +subset: ellipsis_n_bar_2 +templates: + 2ebb68c0-d2f1-49d1-b4e0-74b8dae0c58f: !Template + answer_choices: 1 ||| 2 + id: 2ebb68c0-d2f1-49d1-b4e0-74b8dae0c58f + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_1_2 + reference: '' + 4db2aeb9-59a2-4be6-841e-884a558c6cc6: !Template + answer_choices: A ||| B + id: 4db2aeb9-59a2-4be6-841e-884a558c6cc6 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_A_B + reference: '' + 50f21ae4-7a66-4c3a-81bb-b647f0f2f438: !Template + answer_choices: Yes ||| No + id: 50f21ae4-7a66-4c3a-81bb-b647f0f2f438 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 6e06ad25-f429-4088-b92b-2d4b093ed355: !Template + answer_choices: A ||| B + id: 6e06ad25-f429-4088-b92b-2d4b093ed355 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_A_B + reference: '' + bd29a6c8-dd04-45b5-be84-91936990ef25: !Template + answer_choices: A ||| B + id: bd29a6c8-dd04-45b5-be84-91936990ef25 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: better_A_B + reference: '' + dad6d138-1f7b-4bd4-96a6-288eaf1a5410: !Template + answer_choices: Yes ||| No + id: dad6d138-1f7b-4bd4-96a6-288eaf1a5410 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + e127adfc-bc9f-4a25-a41c-fcbedd434fdc: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: e127adfc-bc9f-4a25-a41c-fcbedd434fdc + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + f74eef44-92da-4edd-86c2-110afd7ba961: !Template + answer_choices: 1 ||| 2 + id: f74eef44-92da-4edd-86c2-110afd7ba961 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_1_2 + reference: '' diff --git a/promptsource/templates/blimp/existential_there_object_raising/templates.yaml b/promptsource/templates/blimp/existential_there_object_raising/templates.yaml new file mode 100644 index 000000000..7c5ed99da --- /dev/null +++ b/promptsource/templates/blimp/existential_there_object_raising/templates.yaml @@ -0,0 +1,262 @@ +dataset: blimp +subset: existential_there_object_raising +templates: + 07617b10-d108-4375-9874-85b565cf0db2: !Template + answer_choices: Yes ||| No + id: 07617b10-d108-4375-9874-85b565cf0db2 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 1b94b2ad-1ffe-404b-bf37-ab0c31c975d1: !Template + answer_choices: A ||| B + id: 1b94b2ad-1ffe-404b-bf37-ab0c31c975d1 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_A_B + reference: '' + 4aacb52f-f2ba-4e54-b8b5-1f6befe8a5d3: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 4aacb52f-f2ba-4e54-b8b5-1f6befe8a5d3 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + 635017b9-8121-48f5-884b-09cba33bd29d: !Template + answer_choices: A ||| B + id: 635017b9-8121-48f5-884b-09cba33bd29d + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_A_B + reference: '' + 6f0ab739-a964-4da4-aea4-44622fe9c0a3: !Template + answer_choices: A ||| B + id: 6f0ab739-a964-4da4-aea4-44622fe9c0a3 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: better_A_B + reference: '' + 7c30c862-b19f-4ba4-94a4-62b4ffb395e7: !Template + answer_choices: Yes ||| No + id: 7c30c862-b19f-4ba4-94a4-62b4ffb395e7 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 7e5a3718-a1ad-4767-85b5-f81404a6569c: !Template + answer_choices: 1 ||| 2 + id: 7e5a3718-a1ad-4767-85b5-f81404a6569c + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_1_2 + reference: '' + c48bf5c5-d91a-450a-9c4e-038da12fff5a: !Template + answer_choices: 1 ||| 2 + id: c48bf5c5-d91a-450a-9c4e-038da12fff5a + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_1_2 + reference: '' diff --git a/promptsource/templates/blimp/existential_there_quantifiers_1/templates.yaml b/promptsource/templates/blimp/existential_there_quantifiers_1/templates.yaml new file mode 100644 index 000000000..85615e7a4 --- /dev/null +++ b/promptsource/templates/blimp/existential_there_quantifiers_1/templates.yaml @@ -0,0 +1,262 @@ +dataset: blimp +subset: existential_there_quantifiers_1 +templates: + 01399a13-0429-4bc3-9fdd-51f448efb5f8: !Template + answer_choices: 1 ||| 2 + id: 01399a13-0429-4bc3-9fdd-51f448efb5f8 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_1_2 + reference: '' + 2081249e-09b0-43b2-a24d-6ac4ae25eaa7: !Template + answer_choices: Yes ||| No + id: 2081249e-09b0-43b2-a24d-6ac4ae25eaa7 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 33871a3f-d1fd-49cd-b485-011df89d7618: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 33871a3f-d1fd-49cd-b485-011df89d7618 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + 36bed627-38c0-485f-99fc-73f80a254029: !Template + answer_choices: 1 ||| 2 + id: 36bed627-38c0-485f-99fc-73f80a254029 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_1_2 + reference: '' + 48a684cf-775d-47f0-ab0d-f8c1de2f4f32: !Template + answer_choices: Yes ||| No + id: 48a684cf-775d-47f0-ab0d-f8c1de2f4f32 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 84950340-88c5-444f-8e76-e7eab10bfa5a: !Template + answer_choices: A ||| B + id: 84950340-88c5-444f-8e76-e7eab10bfa5a + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: better_A_B + reference: '' + a342b8c6-19f3-47cb-8cc5-cbcb1c2705d7: !Template + answer_choices: A ||| B + id: a342b8c6-19f3-47cb-8cc5-cbcb1c2705d7 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_A_B + reference: '' + e3e3be3c-df43-45ab-8a2d-604f35e7b9df: !Template + answer_choices: A ||| B + id: e3e3be3c-df43-45ab-8a2d-604f35e7b9df + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_A_B + reference: '' diff --git a/promptsource/templates/blimp/existential_there_quantifiers_2/templates.yaml b/promptsource/templates/blimp/existential_there_quantifiers_2/templates.yaml new file mode 100644 index 000000000..b62444963 --- /dev/null +++ b/promptsource/templates/blimp/existential_there_quantifiers_2/templates.yaml @@ -0,0 +1,262 @@ +dataset: blimp +subset: existential_there_quantifiers_2 +templates: + 0ce4c191-56f0-44d2-86fe-732ec42a21e0: !Template + answer_choices: Yes ||| No + id: 0ce4c191-56f0-44d2-86fe-732ec42a21e0 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 1d0b5efd-fa90-4f4a-bbec-35d7c0f3c2f1: !Template + answer_choices: A ||| B + id: 1d0b5efd-fa90-4f4a-bbec-35d7c0f3c2f1 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_A_B + reference: '' + 1e5ddb1b-e417-48a4-a3f1-964f835b4582: !Template + answer_choices: 1 ||| 2 + id: 1e5ddb1b-e417-48a4-a3f1-964f835b4582 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_1_2 + reference: '' + 33071e39-1731-4e3d-b50b-6d465df9b71d: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 33071e39-1731-4e3d-b50b-6d465df9b71d + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + 33784c12-becc-4b00-955f-a0315357730e: !Template + answer_choices: Yes ||| No + id: 33784c12-becc-4b00-955f-a0315357730e + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 4c1264d8-d80b-47c3-b5a2-3d7f3305cc55: !Template + answer_choices: 1 ||| 2 + id: 4c1264d8-d80b-47c3-b5a2-3d7f3305cc55 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_1_2 + reference: '' + 6f07a47c-c8c7-4b88-b389-326f5a58c04a: !Template + answer_choices: A ||| B + id: 6f07a47c-c8c7-4b88-b389-326f5a58c04a + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: better_A_B + reference: '' + 829efbf0-b64d-48e1-a9c4-ffff574d3a88: !Template + answer_choices: A ||| B + id: 829efbf0-b64d-48e1-a9c4-ffff574d3a88 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_A_B + reference: '' diff --git a/promptsource/templates/blimp/existential_there_subject_raising/templates.yaml b/promptsource/templates/blimp/existential_there_subject_raising/templates.yaml new file mode 100644 index 000000000..2e7647dc2 --- /dev/null +++ b/promptsource/templates/blimp/existential_there_subject_raising/templates.yaml @@ -0,0 +1,262 @@ +dataset: blimp +subset: existential_there_subject_raising +templates: + 720eb40e-bd94-4355-a3d7-1a39fcda4d48: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 720eb40e-bd94-4355-a3d7-1a39fcda4d48 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + 769032ce-d6e8-4456-9996-8935747f0749: !Template + answer_choices: Yes ||| No + id: 769032ce-d6e8-4456-9996-8935747f0749 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 9b7742fd-42ea-464e-bf01-ac5b87727f22: !Template + answer_choices: Yes ||| No + id: 9b7742fd-42ea-464e-bf01-ac5b87727f22 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + b1749ae9-3c03-49f2-94f2-db48462aa300: !Template + answer_choices: A ||| B + id: b1749ae9-3c03-49f2-94f2-db48462aa300 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: better_A_B + reference: '' + cc088c7d-20d8-493c-aaec-d94e0956bf57: !Template + answer_choices: 1 ||| 2 + id: cc088c7d-20d8-493c-aaec-d94e0956bf57 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_1_2 + reference: '' + cdb9fa8b-5462-4390-8120-68e9741dcdd1: !Template + answer_choices: A ||| B + id: cdb9fa8b-5462-4390-8120-68e9741dcdd1 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_A_B + reference: '' + d85c1524-e607-41b4-8a5f-9487d16489d7: !Template + answer_choices: 1 ||| 2 + id: d85c1524-e607-41b4-8a5f-9487d16489d7 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_1_2 + reference: '' + de2b6985-e35a-4d2c-8b22-0d2edcc080b3: !Template + answer_choices: A ||| B + id: de2b6985-e35a-4d2c-8b22-0d2edcc080b3 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_A_B + reference: '' diff --git a/promptsource/templates/blimp/expletive_it_object_raising/templates.yaml b/promptsource/templates/blimp/expletive_it_object_raising/templates.yaml new file mode 100644 index 000000000..4f7badbed --- /dev/null +++ b/promptsource/templates/blimp/expletive_it_object_raising/templates.yaml @@ -0,0 +1,262 @@ +dataset: blimp +subset: expletive_it_object_raising +templates: + 237ae5ee-7bf4-4b16-bd21-b0b144f9f4b6: !Template + answer_choices: 1 ||| 2 + id: 237ae5ee-7bf4-4b16-bd21-b0b144f9f4b6 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_1_2 + reference: '' + 33021819-c858-4356-99e2-9b3ab1b75f0f: !Template + answer_choices: Yes ||| No + id: 33021819-c858-4356-99e2-9b3ab1b75f0f + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 54db2e29-0f71-47df-bf44-a117ecb387c9: !Template + answer_choices: Yes ||| No + id: 54db2e29-0f71-47df-bf44-a117ecb387c9 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 8be709a2-9bc6-4ac8-8171-bd33e944805a: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 8be709a2-9bc6-4ac8-8171-bd33e944805a + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + a4498ce0-f686-42d1-abbb-1880a09d6dc6: !Template + answer_choices: A ||| B + id: a4498ce0-f686-42d1-abbb-1880a09d6dc6 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_A_B + reference: '' + b05ebbc4-9c71-4a41-9391-a8d82c851671: !Template + answer_choices: 1 ||| 2 + id: b05ebbc4-9c71-4a41-9391-a8d82c851671 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_1_2 + reference: '' + bcd23eec-fd90-4e8f-bb83-523d67e51b4d: !Template + answer_choices: A ||| B + id: bcd23eec-fd90-4e8f-bb83-523d67e51b4d + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: better_A_B + reference: '' + e8e7d70e-4028-4c24-9d29-2cd404252867: !Template + answer_choices: A ||| B + id: e8e7d70e-4028-4c24-9d29-2cd404252867 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_A_B + reference: '' diff --git a/promptsource/templates/blimp/inchoative/templates.yaml b/promptsource/templates/blimp/inchoative/templates.yaml new file mode 100644 index 000000000..eca4118ac --- /dev/null +++ b/promptsource/templates/blimp/inchoative/templates.yaml @@ -0,0 +1,262 @@ +dataset: blimp +subset: inchoative +templates: + 17d36910-7d7f-48e8-8841-a67f594279c3: !Template + answer_choices: 1 ||| 2 + id: 17d36910-7d7f-48e8-8841-a67f594279c3 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_1_2 + reference: '' + 4d108f13-5ba2-48a5-832d-47e7c67c15f8: !Template + answer_choices: Yes ||| No + id: 4d108f13-5ba2-48a5-832d-47e7c67c15f8 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 54dc5d50-fc3f-48d7-9c9d-269eeae24c71: !Template + answer_choices: A ||| B + id: 54dc5d50-fc3f-48d7-9c9d-269eeae24c71 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_A_B + reference: '' + 65029c1f-8921-42e2-92bd-ac997cf249de: !Template + answer_choices: A ||| B + id: 65029c1f-8921-42e2-92bd-ac997cf249de + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_A_B + reference: '' + 76b778db-8193-44bf-b89d-357bce08373a: !Template + answer_choices: 1 ||| 2 + id: 76b778db-8193-44bf-b89d-357bce08373a + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_1_2 + reference: '' + 7db4bd8d-42c0-44da-abb5-7228b128f0d3: !Template + answer_choices: A ||| B + id: 7db4bd8d-42c0-44da-abb5-7228b128f0d3 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: better_A_B + reference: '' + c5de2c20-822b-4676-8645-8f7140c224c8: !Template + answer_choices: Yes ||| No + id: c5de2c20-822b-4676-8645-8f7140c224c8 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + e7765e43-44c5-487c-a968-4cc931598e34: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: e7765e43-44c5-487c-a968-4cc931598e34 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' diff --git a/promptsource/templates/blimp/intransitive/templates.yaml b/promptsource/templates/blimp/intransitive/templates.yaml new file mode 100644 index 000000000..fb8fc6748 --- /dev/null +++ b/promptsource/templates/blimp/intransitive/templates.yaml @@ -0,0 +1,262 @@ +dataset: blimp +subset: intransitive +templates: + 09eccac1-7d0d-4cbf-839f-3815d6b35c3a: !Template + answer_choices: Yes ||| No + id: 09eccac1-7d0d-4cbf-839f-3815d6b35c3a + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 29b6507a-dab9-4e6e-b644-f6896d67091a: !Template + answer_choices: 1 ||| 2 + id: 29b6507a-dab9-4e6e-b644-f6896d67091a + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_1_2 + reference: '' + 4f61d91b-ef03-4700-80be-74c719446815: !Template + answer_choices: A ||| B + id: 4f61d91b-ef03-4700-80be-74c719446815 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: better_A_B + reference: '' + 7d082f4d-f8cb-407b-92bc-18b0ea5bb8d1: !Template + answer_choices: A ||| B + id: 7d082f4d-f8cb-407b-92bc-18b0ea5bb8d1 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_A_B + reference: '' + 82e180ac-5afe-49cc-afb2-8b4dd4c609c1: !Template + answer_choices: Yes ||| No + id: 82e180ac-5afe-49cc-afb2-8b4dd4c609c1 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 947578c4-143a-4062-a476-e9a39f7f8020: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 947578c4-143a-4062-a476-e9a39f7f8020 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + a5fa1d8e-cd6e-448a-9b97-a3331594a9aa: !Template + answer_choices: 1 ||| 2 + id: a5fa1d8e-cd6e-448a-9b97-a3331594a9aa + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_1_2 + reference: '' + bc749983-4028-4466-947b-bc021f616fa0: !Template + answer_choices: A ||| B + id: bc749983-4028-4466-947b-bc021f616fa0 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_A_B + reference: '' diff --git a/promptsource/templates/blimp/irregular_past_participle_adjectives/templates.yaml b/promptsource/templates/blimp/irregular_past_participle_adjectives/templates.yaml new file mode 100644 index 000000000..f51ab26e6 --- /dev/null +++ b/promptsource/templates/blimp/irregular_past_participle_adjectives/templates.yaml @@ -0,0 +1,262 @@ +dataset: blimp +subset: irregular_past_participle_adjectives +templates: + 01c8bd41-9a1f-4030-a4ab-4b21ac314a21: !Template + answer_choices: 1 ||| 2 + id: 01c8bd41-9a1f-4030-a4ab-4b21ac314a21 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_1_2 + reference: '' + 20ceb8f1-af0f-44eb-b06c-7a64715d4165: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 20ceb8f1-af0f-44eb-b06c-7a64715d4165 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + 3b5e3a7e-a847-4a20-af6e-ae94776f15a9: !Template + answer_choices: Yes ||| No + id: 3b5e3a7e-a847-4a20-af6e-ae94776f15a9 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 4a4440cb-a0ba-49f8-89b8-44f29a0abaac: !Template + answer_choices: A ||| B + id: 4a4440cb-a0ba-49f8-89b8-44f29a0abaac + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: better_A_B + reference: '' + 60ecd607-6fa1-4163-b6e4-15ae731edad5: !Template + answer_choices: Yes ||| No + id: 60ecd607-6fa1-4163-b6e4-15ae731edad5 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 7c796d3f-205d-444c-a701-57d52ff9fc4a: !Template + answer_choices: A ||| B + id: 7c796d3f-205d-444c-a701-57d52ff9fc4a + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_A_B + reference: '' + b0716af9-24a4-4d00-8be6-7c61de12bdf2: !Template + answer_choices: A ||| B + id: b0716af9-24a4-4d00-8be6-7c61de12bdf2 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_A_B + reference: '' + d9490bb4-8fb6-458d-94fc-c8918d851240: !Template + answer_choices: 1 ||| 2 + id: d9490bb4-8fb6-458d-94fc-c8918d851240 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_1_2 + reference: '' diff --git a/promptsource/templates/blimp/irregular_past_participle_verbs/templates.yaml b/promptsource/templates/blimp/irregular_past_participle_verbs/templates.yaml new file mode 100644 index 000000000..54e0950ed --- /dev/null +++ b/promptsource/templates/blimp/irregular_past_participle_verbs/templates.yaml @@ -0,0 +1,262 @@ +dataset: blimp +subset: irregular_past_participle_verbs +templates: + 033b7e11-463c-4a4f-a7e4-3010e8042b63: !Template + answer_choices: A ||| B + id: 033b7e11-463c-4a4f-a7e4-3010e8042b63 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_A_B + reference: '' + 094ad547-7da1-4405-b8c4-232b68406bb6: !Template + answer_choices: A ||| B + id: 094ad547-7da1-4405-b8c4-232b68406bb6 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: better_A_B + reference: '' + 1ca9115d-82a3-4e9a-80ce-4b095b391f6c: !Template + answer_choices: 1 ||| 2 + id: 1ca9115d-82a3-4e9a-80ce-4b095b391f6c + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_1_2 + reference: '' + 7455d78b-7d69-4c26-af9f-280078cab5c8: !Template + answer_choices: Yes ||| No + id: 7455d78b-7d69-4c26-af9f-280078cab5c8 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 8d7922a1-5922-412a-ba40-7166ca83f863: !Template + answer_choices: 1 ||| 2 + id: 8d7922a1-5922-412a-ba40-7166ca83f863 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_1_2 + reference: '' + a2e0f26c-aea2-4ebf-88bd-b40f9a8015ba: !Template + answer_choices: A ||| B + id: a2e0f26c-aea2-4ebf-88bd-b40f9a8015ba + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_A_B + reference: '' + d309555f-4f32-4288-bfcd-eebcd1dc5507: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: d309555f-4f32-4288-bfcd-eebcd1dc5507 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + e98ee2cf-a98c-460d-bdd2-c967fc750d0f: !Template + answer_choices: Yes ||| No + id: e98ee2cf-a98c-460d-bdd2-c967fc750d0f + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' diff --git a/promptsource/templates/blimp/irregular_plural_subject_verb_agreement_1/templates.yaml b/promptsource/templates/blimp/irregular_plural_subject_verb_agreement_1/templates.yaml new file mode 100644 index 000000000..5efd68937 --- /dev/null +++ b/promptsource/templates/blimp/irregular_plural_subject_verb_agreement_1/templates.yaml @@ -0,0 +1,262 @@ +dataset: blimp +subset: irregular_plural_subject_verb_agreement_1 +templates: + 18e38f65-c6c5-4496-a5da-38b8f10403f6: !Template + answer_choices: Yes ||| No + id: 18e38f65-c6c5-4496-a5da-38b8f10403f6 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 1ac85c8f-7341-41e8-b9ff-4639a6be27c2: !Template + answer_choices: A ||| B + id: 1ac85c8f-7341-41e8-b9ff-4639a6be27c2 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_A_B + reference: '' + 2530993f-5703-40af-bfba-0dcc3ee8a8f7: !Template + answer_choices: A ||| B + id: 2530993f-5703-40af-bfba-0dcc3ee8a8f7 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: better_A_B + reference: '' + 3f258e68-f24f-4679-a1b0-c67d1af2d346: !Template + answer_choices: 1 ||| 2 + id: 3f258e68-f24f-4679-a1b0-c67d1af2d346 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_1_2 + reference: '' + 78ef0745-70c5-4aeb-a29f-231a15e88842: !Template + answer_choices: 1 ||| 2 + id: 78ef0745-70c5-4aeb-a29f-231a15e88842 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_1_2 + reference: '' + d5e3d605-0b6d-4a2a-a75e-12fede0bc849: !Template + answer_choices: Yes ||| No + id: d5e3d605-0b6d-4a2a-a75e-12fede0bc849 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + dbfdec5c-9a34-4121-81d9-61978296d0fb: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: dbfdec5c-9a34-4121-81d9-61978296d0fb + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + f3c61507-8558-4b9d-9c5b-0476563b553a: !Template + answer_choices: A ||| B + id: f3c61507-8558-4b9d-9c5b-0476563b553a + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_A_B + reference: '' diff --git a/promptsource/templates/blimp/irregular_plural_subject_verb_agreement_2/templates.yaml b/promptsource/templates/blimp/irregular_plural_subject_verb_agreement_2/templates.yaml new file mode 100644 index 000000000..469d8aeb7 --- /dev/null +++ b/promptsource/templates/blimp/irregular_plural_subject_verb_agreement_2/templates.yaml @@ -0,0 +1,262 @@ +dataset: blimp +subset: irregular_plural_subject_verb_agreement_2 +templates: + 6a203fde-56e9-42bb-bae9-639748ff420a: !Template + answer_choices: Yes ||| No + id: 6a203fde-56e9-42bb-bae9-639748ff420a + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 95c16bb7-58e4-499f-a4cb-aec35dbb5505: !Template + answer_choices: 1 ||| 2 + id: 95c16bb7-58e4-499f-a4cb-aec35dbb5505 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_1_2 + reference: '' + a0ef739a-6f04-4529-ae50-cc146355e526: !Template + answer_choices: A ||| B + id: a0ef739a-6f04-4529-ae50-cc146355e526 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: better_A_B + reference: '' + af407614-1c7b-424f-acfe-37ab790703f3: !Template + answer_choices: 1 ||| 2 + id: af407614-1c7b-424f-acfe-37ab790703f3 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_1_2 + reference: '' + ca6d5112-5d25-4611-9886-79618bba1afa: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: ca6d5112-5d25-4611-9886-79618bba1afa + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + d7b194b9-39c6-4ae8-a85f-d8bdb961bc93: !Template + answer_choices: A ||| B + id: d7b194b9-39c6-4ae8-a85f-d8bdb961bc93 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_A_B + reference: '' + eea5f683-71fc-4660-a844-ab5ea175c740: !Template + answer_choices: A ||| B + id: eea5f683-71fc-4660-a844-ab5ea175c740 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_A_B + reference: '' + f8ac205b-8848-4c70-b13b-7148f3c09294: !Template + answer_choices: Yes ||| No + id: f8ac205b-8848-4c70-b13b-7148f3c09294 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' diff --git a/promptsource/templates/blimp/left_branch_island_echo_question/templates.yaml b/promptsource/templates/blimp/left_branch_island_echo_question/templates.yaml new file mode 100644 index 000000000..f10cc606e --- /dev/null +++ b/promptsource/templates/blimp/left_branch_island_echo_question/templates.yaml @@ -0,0 +1,262 @@ +dataset: blimp +subset: left_branch_island_echo_question +templates: + 309c0c8a-f7b0-475a-8fd0-65fba3ece1bb: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 309c0c8a-f7b0-475a-8fd0-65fba3ece1bb + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + 7da51f1d-f891-48e6-b271-b9602d7eb08b: !Template + answer_choices: Yes ||| No + id: 7da51f1d-f891-48e6-b271-b9602d7eb08b + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + c0402910-d21b-4dc2-ab00-6823d349c3a8: !Template + answer_choices: Yes ||| No + id: c0402910-d21b-4dc2-ab00-6823d349c3a8 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + e10efdbb-6452-4446-befd-42bff2da8d51: !Template + answer_choices: 1 ||| 2 + id: e10efdbb-6452-4446-befd-42bff2da8d51 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_1_2 + reference: '' + e7a31ef5-2205-4deb-a5d7-6d872b85f55d: !Template + answer_choices: A ||| B + id: e7a31ef5-2205-4deb-a5d7-6d872b85f55d + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_A_B + reference: '' + f052b636-8fdc-45e4-aaad-10147b6e28be: !Template + answer_choices: A ||| B + id: f052b636-8fdc-45e4-aaad-10147b6e28be + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: better_A_B + reference: '' + f61cf914-d4d2-47d5-bc18-4196932de519: !Template + answer_choices: 1 ||| 2 + id: f61cf914-d4d2-47d5-bc18-4196932de519 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_1_2 + reference: '' + fafa04c4-3b6a-49fb-9cb2-7a1e6f2285be: !Template + answer_choices: A ||| B + id: fafa04c4-3b6a-49fb-9cb2-7a1e6f2285be + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_A_B + reference: '' diff --git a/promptsource/templates/blimp/left_branch_island_simple_question/templates.yaml b/promptsource/templates/blimp/left_branch_island_simple_question/templates.yaml new file mode 100644 index 000000000..4fa8d5b05 --- /dev/null +++ b/promptsource/templates/blimp/left_branch_island_simple_question/templates.yaml @@ -0,0 +1,262 @@ +dataset: blimp +subset: left_branch_island_simple_question +templates: + 12dc4d87-ea95-4f68-ac86-b6b17d5448b5: !Template + answer_choices: A ||| B + id: 12dc4d87-ea95-4f68-ac86-b6b17d5448b5 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_A_B + reference: '' + 1fcf9d58-92d1-4480-a4f3-01cc0421ff05: !Template + answer_choices: A ||| B + id: 1fcf9d58-92d1-4480-a4f3-01cc0421ff05 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: better_A_B + reference: '' + 50e1352f-74ff-40db-bf40-228a863405d3: !Template + answer_choices: Yes ||| No + id: 50e1352f-74ff-40db-bf40-228a863405d3 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + b559f92f-12d3-4dd5-afe4-768a0bdb6aaf: !Template + answer_choices: A ||| B + id: b559f92f-12d3-4dd5-afe4-768a0bdb6aaf + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_A_B + reference: '' + d3dd8509-70fc-4937-aca6-3750274307ca: !Template + answer_choices: 1 ||| 2 + id: d3dd8509-70fc-4937-aca6-3750274307ca + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_1_2 + reference: '' + e214c1ce-5271-4d7b-99b2-70b31e4d6da0: !Template + answer_choices: Yes ||| No + id: e214c1ce-5271-4d7b-99b2-70b31e4d6da0 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + f8ca733a-5639-4a48-b109-87e1c000ea86: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: f8ca733a-5639-4a48-b109-87e1c000ea86 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + fe09912e-a883-4b7d-b11f-9c65b8295a4e: !Template + answer_choices: 1 ||| 2 + id: fe09912e-a883-4b7d-b11f-9c65b8295a4e + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_1_2 + reference: '' diff --git a/promptsource/templates/blimp/matrix_question_npi_licensor_present/templates.yaml b/promptsource/templates/blimp/matrix_question_npi_licensor_present/templates.yaml new file mode 100644 index 000000000..382f70bdc --- /dev/null +++ b/promptsource/templates/blimp/matrix_question_npi_licensor_present/templates.yaml @@ -0,0 +1,262 @@ +dataset: blimp +subset: matrix_question_npi_licensor_present +templates: + 059851a0-7f9d-43e8-a131-bbdb88d2f68f: !Template + answer_choices: A ||| B + id: 059851a0-7f9d-43e8-a131-bbdb88d2f68f + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_A_B + reference: '' + 25a49a3a-0826-4b55-8638-f2168bcd80b9: !Template + answer_choices: Yes ||| No + id: 25a49a3a-0826-4b55-8638-f2168bcd80b9 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 2ff72c8a-e156-4cf3-9522-9d9a338a3e50: !Template + answer_choices: 1 ||| 2 + id: 2ff72c8a-e156-4cf3-9522-9d9a338a3e50 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_1_2 + reference: '' + a85cb8d7-76a9-4024-b63e-cfcfd00bea67: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: a85cb8d7-76a9-4024-b63e-cfcfd00bea67 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + af64a7ec-3c17-42f6-a183-9aa63c2c1d3e: !Template + answer_choices: A ||| B + id: af64a7ec-3c17-42f6-a183-9aa63c2c1d3e + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_A_B + reference: '' + d0748d18-8341-4e1d-b462-2a05c6bf2996: !Template + answer_choices: Yes ||| No + id: d0748d18-8341-4e1d-b462-2a05c6bf2996 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + eab99bb5-b797-46d7-a075-eb416594cdc9: !Template + answer_choices: A ||| B + id: eab99bb5-b797-46d7-a075-eb416594cdc9 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: better_A_B + reference: '' + ec11e437-3559-48bc-b066-abb789d43a98: !Template + answer_choices: 1 ||| 2 + id: ec11e437-3559-48bc-b066-abb789d43a98 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_1_2 + reference: '' diff --git a/promptsource/templates/blimp/npi_present_1/templates.yaml b/promptsource/templates/blimp/npi_present_1/templates.yaml new file mode 100644 index 000000000..2a9967746 --- /dev/null +++ b/promptsource/templates/blimp/npi_present_1/templates.yaml @@ -0,0 +1,262 @@ +dataset: blimp +subset: npi_present_1 +templates: + 39a7b62a-0043-4f86-99c4-8a74e7e6ed9d: !Template + answer_choices: A ||| B + id: 39a7b62a-0043-4f86-99c4-8a74e7e6ed9d + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_A_B + reference: '' + 419aeb25-4a54-4572-bc52-46e8ae98dd5c: !Template + answer_choices: 1 ||| 2 + id: 419aeb25-4a54-4572-bc52-46e8ae98dd5c + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_1_2 + reference: '' + 6c628463-ca54-43ff-b847-ea73eb9184f7: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 6c628463-ca54-43ff-b847-ea73eb9184f7 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + 7f04187c-ac4e-474a-8e0e-d5ae69665383: !Template + answer_choices: A ||| B + id: 7f04187c-ac4e-474a-8e0e-d5ae69665383 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_A_B + reference: '' + 8f87db45-1335-49f2-8d61-0cb245a33671: !Template + answer_choices: Yes ||| No + id: 8f87db45-1335-49f2-8d61-0cb245a33671 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + bd46ec4a-98b0-4775-8383-c0118161c9fa: !Template + answer_choices: 1 ||| 2 + id: bd46ec4a-98b0-4775-8383-c0118161c9fa + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_1_2 + reference: '' + ed1b7e89-2e19-4b67-bb0d-6a9b3c1a3c1f: !Template + answer_choices: Yes ||| No + id: ed1b7e89-2e19-4b67-bb0d-6a9b3c1a3c1f + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + f30fa011-e77c-450e-8771-c10b10f2f367: !Template + answer_choices: A ||| B + id: f30fa011-e77c-450e-8771-c10b10f2f367 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: better_A_B + reference: '' diff --git a/promptsource/templates/blimp/npi_present_2/templates.yaml b/promptsource/templates/blimp/npi_present_2/templates.yaml new file mode 100644 index 000000000..37e6695c0 --- /dev/null +++ b/promptsource/templates/blimp/npi_present_2/templates.yaml @@ -0,0 +1,262 @@ +dataset: blimp +subset: npi_present_2 +templates: + 1ef38606-ee86-42e3-acab-d8406b4d4124: !Template + answer_choices: Yes ||| No + id: 1ef38606-ee86-42e3-acab-d8406b4d4124 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 2157e41e-8ad7-466b-ba8d-49ac6fa544ed: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 2157e41e-8ad7-466b-ba8d-49ac6fa544ed + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + 21bc7e99-6dff-46e0-8eb5-04256f170dfc: !Template + answer_choices: 1 ||| 2 + id: 21bc7e99-6dff-46e0-8eb5-04256f170dfc + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_1_2 + reference: '' + 24d2dc86-e2f3-4537-8c6c-d9b119208140: !Template + answer_choices: 1 ||| 2 + id: 24d2dc86-e2f3-4537-8c6c-d9b119208140 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_1_2 + reference: '' + 63a36bd9-8f58-4c79-8965-c1b76d461849: !Template + answer_choices: Yes ||| No + id: 63a36bd9-8f58-4c79-8965-c1b76d461849 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + b8864770-3d48-4499-a1b4-81fa66764a1c: !Template + answer_choices: A ||| B + id: b8864770-3d48-4499-a1b4-81fa66764a1c + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_A_B + reference: '' + c5bc3e1e-f756-4bff-86e8-bfefd6eaa937: !Template + answer_choices: A ||| B + id: c5bc3e1e-f756-4bff-86e8-bfefd6eaa937 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_A_B + reference: '' + e93bdcbe-887b-4935-9e9d-61662829622a: !Template + answer_choices: A ||| B + id: e93bdcbe-887b-4935-9e9d-61662829622a + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: better_A_B + reference: '' diff --git a/promptsource/templates/blimp/only_npi_licensor_present/templates.yaml b/promptsource/templates/blimp/only_npi_licensor_present/templates.yaml new file mode 100644 index 000000000..5c42308f8 --- /dev/null +++ b/promptsource/templates/blimp/only_npi_licensor_present/templates.yaml @@ -0,0 +1,262 @@ +dataset: blimp +subset: only_npi_licensor_present +templates: + 24fa314d-7c8f-49e2-a684-e5ad128e0e7d: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 24fa314d-7c8f-49e2-a684-e5ad128e0e7d + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + 66871ba2-5e1a-4d07-a10e-793bfed083c2: !Template + answer_choices: A ||| B + id: 66871ba2-5e1a-4d07-a10e-793bfed083c2 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_A_B + reference: '' + 6ff97bae-77eb-4619-940a-8c529d8c4d08: !Template + answer_choices: Yes ||| No + id: 6ff97bae-77eb-4619-940a-8c529d8c4d08 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 8e80598f-4497-4a5f-bbd4-e37f93460af6: !Template + answer_choices: Yes ||| No + id: 8e80598f-4497-4a5f-bbd4-e37f93460af6 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + a7db897c-bcea-4726-aaf4-07502d8cac5c: !Template + answer_choices: 1 ||| 2 + id: a7db897c-bcea-4726-aaf4-07502d8cac5c + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_1_2 + reference: '' + b7a9dcd3-fd60-412d-b0eb-7dd7365c97ab: !Template + answer_choices: A ||| B + id: b7a9dcd3-fd60-412d-b0eb-7dd7365c97ab + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_A_B + reference: '' + b8bf85da-cc09-4c2c-adbf-8013785c6e23: !Template + answer_choices: 1 ||| 2 + id: b8bf85da-cc09-4c2c-adbf-8013785c6e23 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_1_2 + reference: '' + c7b622be-51e2-4579-97b6-bf6500621aa4: !Template + answer_choices: A ||| B + id: c7b622be-51e2-4579-97b6-bf6500621aa4 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: better_A_B + reference: '' diff --git a/promptsource/templates/blimp/only_npi_scope/templates.yaml b/promptsource/templates/blimp/only_npi_scope/templates.yaml new file mode 100644 index 000000000..caee2be2b --- /dev/null +++ b/promptsource/templates/blimp/only_npi_scope/templates.yaml @@ -0,0 +1,262 @@ +dataset: blimp +subset: only_npi_scope +templates: + 05e40054-b360-48a3-84c3-82da5de36bc7: !Template + answer_choices: Yes ||| No + id: 05e40054-b360-48a3-84c3-82da5de36bc7 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 169f2bdf-d96c-4614-a3c2-23063c5109d5: !Template + answer_choices: Yes ||| No + id: 169f2bdf-d96c-4614-a3c2-23063c5109d5 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 3511bc99-c76d-4779-8edd-db0e9d07279b: !Template + answer_choices: A ||| B + id: 3511bc99-c76d-4779-8edd-db0e9d07279b + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: better_A_B + reference: '' + 37ec526e-e774-4707-81c4-8f934c0f8e56: !Template + answer_choices: A ||| B + id: 37ec526e-e774-4707-81c4-8f934c0f8e56 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_A_B + reference: '' + 8cb778e7-9143-41b5-b5cd-4861e5fd541e: !Template + answer_choices: 1 ||| 2 + id: 8cb778e7-9143-41b5-b5cd-4861e5fd541e + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_1_2 + reference: '' + be758fd8-40e8-45d7-a5a7-a58ec468d540: !Template + answer_choices: 1 ||| 2 + id: be758fd8-40e8-45d7-a5a7-a58ec468d540 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_1_2 + reference: '' + c2f57af5-3e4c-4a3d-8ed3-48ff5bcd84e8: !Template + answer_choices: A ||| B + id: c2f57af5-3e4c-4a3d-8ed3-48ff5bcd84e8 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_A_B + reference: '' + dcaf199f-bf2e-4c60-ba9e-c542574c33cf: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: dcaf199f-bf2e-4c60-ba9e-c542574c33cf + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' diff --git a/promptsource/templates/blimp/passive_1/templates.yaml b/promptsource/templates/blimp/passive_1/templates.yaml new file mode 100644 index 000000000..5a2fa9bcc --- /dev/null +++ b/promptsource/templates/blimp/passive_1/templates.yaml @@ -0,0 +1,262 @@ +dataset: blimp +subset: passive_1 +templates: + 04fda615-27b1-465e-9440-4c96b9b388fc: !Template + answer_choices: 1 ||| 2 + id: 04fda615-27b1-465e-9440-4c96b9b388fc + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_1_2 + reference: '' + 0602ba7a-91b1-4054-940b-84f5873397c6: !Template + answer_choices: Yes ||| No + id: 0602ba7a-91b1-4054-940b-84f5873397c6 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 44cdd60a-bd52-46d5-9c17-f1784ea50262: !Template + answer_choices: A ||| B + id: 44cdd60a-bd52-46d5-9c17-f1784ea50262 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_A_B + reference: '' + 59071832-47f0-403f-9b84-8ee181a470ad: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 59071832-47f0-403f-9b84-8ee181a470ad + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + 737871d0-ec0b-4295-bd8c-cc9a6c39e208: !Template + answer_choices: A ||| B + id: 737871d0-ec0b-4295-bd8c-cc9a6c39e208 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: better_A_B + reference: '' + 7fd8760e-3c98-49d3-b508-a5547fbb18aa: !Template + answer_choices: A ||| B + id: 7fd8760e-3c98-49d3-b508-a5547fbb18aa + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_A_B + reference: '' + b0d0957e-e73a-4019-ad55-8165f918c5a0: !Template + answer_choices: 1 ||| 2 + id: b0d0957e-e73a-4019-ad55-8165f918c5a0 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_1_2 + reference: '' + d064c4af-e701-43a8-9bff-5699dc39d8f8: !Template + answer_choices: Yes ||| No + id: d064c4af-e701-43a8-9bff-5699dc39d8f8 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' diff --git a/promptsource/templates/blimp/passive_2/templates.yaml b/promptsource/templates/blimp/passive_2/templates.yaml new file mode 100644 index 000000000..e4225391e --- /dev/null +++ b/promptsource/templates/blimp/passive_2/templates.yaml @@ -0,0 +1,262 @@ +dataset: blimp +subset: passive_2 +templates: + 06b9fe19-f65a-4058-b92e-e3bde058d82e: !Template + answer_choices: 1 ||| 2 + id: 06b9fe19-f65a-4058-b92e-e3bde058d82e + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_1_2 + reference: '' + 2c585c96-1c50-4695-8aef-d6abfb42dcd7: !Template + answer_choices: Yes ||| No + id: 2c585c96-1c50-4695-8aef-d6abfb42dcd7 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 7c20c5d7-a548-4644-8a0f-5e926105b8b7: !Template + answer_choices: A ||| B + id: 7c20c5d7-a548-4644-8a0f-5e926105b8b7 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: better_A_B + reference: '' + b973a531-bfbb-40fd-abc2-1d68ae7e91d3: !Template + answer_choices: 1 ||| 2 + id: b973a531-bfbb-40fd-abc2-1d68ae7e91d3 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_1_2 + reference: '' + c6a996bf-1528-4276-bc92-d8655e2d2449: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: c6a996bf-1528-4276-bc92-d8655e2d2449 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + d1463fda-d34e-40b6-bca3-e6322145f5fd: !Template + answer_choices: A ||| B + id: d1463fda-d34e-40b6-bca3-e6322145f5fd + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_A_B + reference: '' + d47cdd9f-f846-4731-bb25-ab51ead0fa91: !Template + answer_choices: A ||| B + id: d47cdd9f-f846-4731-bb25-ab51ead0fa91 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_A_B + reference: '' + d95f71d4-9c2b-4539-9710-3121834b56cb: !Template + answer_choices: Yes ||| No + id: d95f71d4-9c2b-4539-9710-3121834b56cb + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' diff --git a/promptsource/templates/blimp/principle_A_c_command/templates.yaml b/promptsource/templates/blimp/principle_A_c_command/templates.yaml new file mode 100644 index 000000000..8374e386a --- /dev/null +++ b/promptsource/templates/blimp/principle_A_c_command/templates.yaml @@ -0,0 +1,262 @@ +dataset: blimp +subset: principle_A_c_command +templates: + 2b0ef285-4e29-4c9c-bd17-a0e7942b007d: !Template + answer_choices: 1 ||| 2 + id: 2b0ef285-4e29-4c9c-bd17-a0e7942b007d + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_1_2 + reference: '' + 4062dcf3-d7fe-4fd9-841c-1aec645cfa47: !Template + answer_choices: Yes ||| No + id: 4062dcf3-d7fe-4fd9-841c-1aec645cfa47 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 642f56eb-0e53-4cc0-96b4-4a5460c89c43: !Template + answer_choices: A ||| B + id: 642f56eb-0e53-4cc0-96b4-4a5460c89c43 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: better_A_B + reference: '' + 77857193-d7f8-41b1-bae9-3ea8c1b8a4e4: !Template + answer_choices: A ||| B + id: 77857193-d7f8-41b1-bae9-3ea8c1b8a4e4 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_A_B + reference: '' + 78b70ffb-a7c0-4196-810d-0b7886172c28: !Template + answer_choices: 1 ||| 2 + id: 78b70ffb-a7c0-4196-810d-0b7886172c28 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_1_2 + reference: '' + 7f543e11-9bbc-4175-9fcc-2cdeefd024bf: !Template + answer_choices: Yes ||| No + id: 7f543e11-9bbc-4175-9fcc-2cdeefd024bf + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + b56211bb-d767-4494-a0b4-77c66f0c67b4: !Template + answer_choices: A ||| B + id: b56211bb-d767-4494-a0b4-77c66f0c67b4 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_A_B + reference: '' + de3b026d-d470-4074-9271-39b04e28f29a: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: de3b026d-d470-4074-9271-39b04e28f29a + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' diff --git a/promptsource/templates/blimp/principle_A_case_1/templates.yaml b/promptsource/templates/blimp/principle_A_case_1/templates.yaml new file mode 100644 index 000000000..4f057672e --- /dev/null +++ b/promptsource/templates/blimp/principle_A_case_1/templates.yaml @@ -0,0 +1,262 @@ +dataset: blimp +subset: principle_A_case_1 +templates: + 3779766b-d75e-4a0c-8019-829046811ed5: !Template + answer_choices: 1 ||| 2 + id: 3779766b-d75e-4a0c-8019-829046811ed5 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_1_2 + reference: '' + 49d9d4c4-0be1-4bf8-be40-42d05dd6829b: !Template + answer_choices: Yes ||| No + id: 49d9d4c4-0be1-4bf8-be40-42d05dd6829b + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 7450e2e8-9bbd-486d-b857-7602012f8600: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 7450e2e8-9bbd-486d-b857-7602012f8600 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + 8939f24b-7a3e-4a91-bda2-c428c2ca758c: !Template + answer_choices: Yes ||| No + id: 8939f24b-7a3e-4a91-bda2-c428c2ca758c + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 9675ffe4-e45b-413b-abb2-99d0af78ab29: !Template + answer_choices: 1 ||| 2 + id: 9675ffe4-e45b-413b-abb2-99d0af78ab29 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_1_2 + reference: '' + aed1b5f9-a572-4174-84aa-0d298753bd41: !Template + answer_choices: A ||| B + id: aed1b5f9-a572-4174-84aa-0d298753bd41 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: better_A_B + reference: '' + c64a3a1e-289a-42ac-9979-1bd844da1b5d: !Template + answer_choices: A ||| B + id: c64a3a1e-289a-42ac-9979-1bd844da1b5d + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_A_B + reference: '' + e15f0627-2b98-4bde-947d-a4150428c510: !Template + answer_choices: A ||| B + id: e15f0627-2b98-4bde-947d-a4150428c510 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_A_B + reference: '' diff --git a/promptsource/templates/blimp/principle_A_case_2/templates.yaml b/promptsource/templates/blimp/principle_A_case_2/templates.yaml new file mode 100644 index 000000000..804cc90d5 --- /dev/null +++ b/promptsource/templates/blimp/principle_A_case_2/templates.yaml @@ -0,0 +1,262 @@ +dataset: blimp +subset: principle_A_case_2 +templates: + 005dc559-848e-4fc5-9798-45bb9542f4b9: !Template + answer_choices: A ||| B + id: 005dc559-848e-4fc5-9798-45bb9542f4b9 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: better_A_B + reference: '' + 01172621-67c3-4bfa-aae8-0580d3b6b125: !Template + answer_choices: A ||| B + id: 01172621-67c3-4bfa-aae8-0580d3b6b125 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_A_B + reference: '' + 2d12c46e-58f0-4353-b6ae-bd9b773b4c18: !Template + answer_choices: Yes ||| No + id: 2d12c46e-58f0-4353-b6ae-bd9b773b4c18 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 540ee3d2-f789-42b3-8614-1d867342159d: !Template + answer_choices: 1 ||| 2 + id: 540ee3d2-f789-42b3-8614-1d867342159d + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_1_2 + reference: '' + 69098dea-4e33-401b-b61a-82a25f551625: !Template + answer_choices: 1 ||| 2 + id: 69098dea-4e33-401b-b61a-82a25f551625 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_1_2 + reference: '' + 7ed4158d-5939-4b71-8579-a836c4e9158c: !Template + answer_choices: A ||| B + id: 7ed4158d-5939-4b71-8579-a836c4e9158c + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_A_B + reference: '' + a065510f-7097-45de-be3e-0b3f78b887e2: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: a065510f-7097-45de-be3e-0b3f78b887e2 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + df6ebc86-44be-4a68-9c1b-4a1246099eb8: !Template + answer_choices: Yes ||| No + id: df6ebc86-44be-4a68-9c1b-4a1246099eb8 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' diff --git a/promptsource/templates/blimp/principle_A_domain_1/templates.yaml b/promptsource/templates/blimp/principle_A_domain_1/templates.yaml new file mode 100644 index 000000000..6a4b1e49f --- /dev/null +++ b/promptsource/templates/blimp/principle_A_domain_1/templates.yaml @@ -0,0 +1,262 @@ +dataset: blimp +subset: principle_A_domain_1 +templates: + 2bbd2f4c-9c2b-41e0-b773-043baca30250: !Template + answer_choices: A ||| B + id: 2bbd2f4c-9c2b-41e0-b773-043baca30250 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_A_B + reference: '' + 3d340473-b40b-4002-91b1-10139368d878: !Template + answer_choices: Yes ||| No + id: 3d340473-b40b-4002-91b1-10139368d878 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 469288fd-bede-42ea-b2e8-de48dd45a444: !Template + answer_choices: Yes ||| No + id: 469288fd-bede-42ea-b2e8-de48dd45a444 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 4e445447-cfd4-44e0-9a28-e82766410da7: !Template + answer_choices: A ||| B + id: 4e445447-cfd4-44e0-9a28-e82766410da7 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_A_B + reference: '' + 78b2287a-4a15-4e7a-9407-4fdcac441979: !Template + answer_choices: 1 ||| 2 + id: 78b2287a-4a15-4e7a-9407-4fdcac441979 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_1_2 + reference: '' + 86b9e874-8e9f-407d-8a9e-1ca59c793a2d: !Template + answer_choices: 1 ||| 2 + id: 86b9e874-8e9f-407d-8a9e-1ca59c793a2d + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_1_2 + reference: '' + cf260efd-b607-4b6b-9089-972c5f680d7f: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: cf260efd-b607-4b6b-9089-972c5f680d7f + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + d7d066d2-6480-4589-a1a4-e39871d2bfda: !Template + answer_choices: A ||| B + id: d7d066d2-6480-4589-a1a4-e39871d2bfda + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: better_A_B + reference: '' diff --git a/promptsource/templates/blimp/principle_A_domain_2/templates.yaml b/promptsource/templates/blimp/principle_A_domain_2/templates.yaml new file mode 100644 index 000000000..923191944 --- /dev/null +++ b/promptsource/templates/blimp/principle_A_domain_2/templates.yaml @@ -0,0 +1,262 @@ +dataset: blimp +subset: principle_A_domain_2 +templates: + 0bc826f5-d7b1-4f6d-b697-ef6ad2c1200c: !Template + answer_choices: Yes ||| No + id: 0bc826f5-d7b1-4f6d-b697-ef6ad2c1200c + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 1b27d629-a2fc-4269-9b5f-80e549ee60fa: !Template + answer_choices: A ||| B + id: 1b27d629-a2fc-4269-9b5f-80e549ee60fa + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: better_A_B + reference: '' + 1ca4027f-6b05-4991-8008-f6ac0dde727f: !Template + answer_choices: A ||| B + id: 1ca4027f-6b05-4991-8008-f6ac0dde727f + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_A_B + reference: '' + 4cb7ba89-4eab-4095-9174-887353bc2986: !Template + answer_choices: 1 ||| 2 + id: 4cb7ba89-4eab-4095-9174-887353bc2986 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_1_2 + reference: '' + 69defef5-befc-450c-b400-3471dac4be8d: !Template + answer_choices: A ||| B + id: 69defef5-befc-450c-b400-3471dac4be8d + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_A_B + reference: '' + b26d04cb-8633-4e6e-a89e-a85a6dce1736: !Template + answer_choices: 1 ||| 2 + id: b26d04cb-8633-4e6e-a89e-a85a6dce1736 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_1_2 + reference: '' + bb641f36-41e6-418b-9cdb-5b04ebf51715: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: bb641f36-41e6-418b-9cdb-5b04ebf51715 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + fe3c752d-dbbb-43fa-90a2-a866b30ea6dc: !Template + answer_choices: Yes ||| No + id: fe3c752d-dbbb-43fa-90a2-a866b30ea6dc + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' diff --git a/promptsource/templates/blimp/principle_A_domain_3/templates.yaml b/promptsource/templates/blimp/principle_A_domain_3/templates.yaml new file mode 100644 index 000000000..da38bba26 --- /dev/null +++ b/promptsource/templates/blimp/principle_A_domain_3/templates.yaml @@ -0,0 +1,262 @@ +dataset: blimp +subset: principle_A_domain_3 +templates: + 062eb369-a1b9-4ae5-8f89-f500ccddb0a8: !Template + answer_choices: A ||| B + id: 062eb369-a1b9-4ae5-8f89-f500ccddb0a8 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: better_A_B + reference: '' + 54dbeaa7-cfcf-40d3-b2a8-a022b79761e4: !Template + answer_choices: 1 ||| 2 + id: 54dbeaa7-cfcf-40d3-b2a8-a022b79761e4 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_1_2 + reference: '' + 580024a2-37e4-4b2b-b8a3-42ce82e95fe8: !Template + answer_choices: A ||| B + id: 580024a2-37e4-4b2b-b8a3-42ce82e95fe8 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_A_B + reference: '' + 72a0475b-45ca-4a3f-a2a9-89be9ba1a1dd: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 72a0475b-45ca-4a3f-a2a9-89be9ba1a1dd + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + a4f146f4-99fb-4461-9d0c-fa53a29d5dae: !Template + answer_choices: Yes ||| No + id: a4f146f4-99fb-4461-9d0c-fa53a29d5dae + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + b9c16798-57fe-4958-82f4-d7d3e508909c: !Template + answer_choices: A ||| B + id: b9c16798-57fe-4958-82f4-d7d3e508909c + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_A_B + reference: '' + eb65f770-e750-4948-879f-654edc782d5f: !Template + answer_choices: Yes ||| No + id: eb65f770-e750-4948-879f-654edc782d5f + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + f00d14a7-110c-4ee6-98e8-24437696bff2: !Template + answer_choices: 1 ||| 2 + id: f00d14a7-110c-4ee6-98e8-24437696bff2 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_1_2 + reference: '' diff --git a/promptsource/templates/blimp/principle_A_reconstruction/templates.yaml b/promptsource/templates/blimp/principle_A_reconstruction/templates.yaml new file mode 100644 index 000000000..5951822d8 --- /dev/null +++ b/promptsource/templates/blimp/principle_A_reconstruction/templates.yaml @@ -0,0 +1,262 @@ +dataset: blimp +subset: principle_A_reconstruction +templates: + 4cf1935d-3ed0-4076-a2fa-d5275392906d: !Template + answer_choices: Yes ||| No + id: 4cf1935d-3ed0-4076-a2fa-d5275392906d + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 7a7e470f-87ba-4f93-bffc-388328701910: !Template + answer_choices: Yes ||| No + id: 7a7e470f-87ba-4f93-bffc-388328701910 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 883a8e51-9890-4ef6-942d-255b47a7d724: !Template + answer_choices: A ||| B + id: 883a8e51-9890-4ef6-942d-255b47a7d724 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: better_A_B + reference: '' + 99fbf5d8-c746-4eb9-81ae-21dec2382ef5: !Template + answer_choices: 1 ||| 2 + id: 99fbf5d8-c746-4eb9-81ae-21dec2382ef5 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_1_2 + reference: '' + abc3a6ea-baac-4f8f-9a7e-f3e4242ecc46: !Template + answer_choices: 1 ||| 2 + id: abc3a6ea-baac-4f8f-9a7e-f3e4242ecc46 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_1_2 + reference: '' + bc16c071-ea83-488e-bc4c-fecd07048451: !Template + answer_choices: A ||| B + id: bc16c071-ea83-488e-bc4c-fecd07048451 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_A_B + reference: '' + e3a2610f-0d79-4f09-a958-05df12dba4e6: !Template + answer_choices: A ||| B + id: e3a2610f-0d79-4f09-a958-05df12dba4e6 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_A_B + reference: '' + ec07a3c1-fbb3-46eb-ab9a-c891941ea8d5: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: ec07a3c1-fbb3-46eb-ab9a-c891941ea8d5 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' diff --git a/promptsource/templates/blimp/regular_plural_subject_verb_agreement_1/templates.yaml b/promptsource/templates/blimp/regular_plural_subject_verb_agreement_1/templates.yaml new file mode 100644 index 000000000..a080404bc --- /dev/null +++ b/promptsource/templates/blimp/regular_plural_subject_verb_agreement_1/templates.yaml @@ -0,0 +1,262 @@ +dataset: blimp +subset: regular_plural_subject_verb_agreement_1 +templates: + 3cb9bff1-d20f-4370-b12e-aa0aba3756b9: !Template + answer_choices: A ||| B + id: 3cb9bff1-d20f-4370-b12e-aa0aba3756b9 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_A_B + reference: '' + 43a65a43-4d6c-4aaa-ae1f-6ea76aa09511: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 43a65a43-4d6c-4aaa-ae1f-6ea76aa09511 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + 584848e5-8a5e-42fc-bf70-433eec10c8f8: !Template + answer_choices: 1 ||| 2 + id: 584848e5-8a5e-42fc-bf70-433eec10c8f8 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_1_2 + reference: '' + 5cc330d7-d3a9-4fd9-bdea-cdee4c31401c: !Template + answer_choices: 1 ||| 2 + id: 5cc330d7-d3a9-4fd9-bdea-cdee4c31401c + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_1_2 + reference: '' + 5e152d33-d1ff-41a0-a762-e2cdc7148ace: !Template + answer_choices: A ||| B + id: 5e152d33-d1ff-41a0-a762-e2cdc7148ace + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: better_A_B + reference: '' + 9da7bddc-fd8b-4fff-a068-5af8d867f529: !Template + answer_choices: Yes ||| No + id: 9da7bddc-fd8b-4fff-a068-5af8d867f529 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + a63d58b4-8dab-4f53-aa13-efa191bc0d0c: !Template + answer_choices: A ||| B + id: a63d58b4-8dab-4f53-aa13-efa191bc0d0c + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_A_B + reference: '' + e83f2ca0-0c6a-4cf6-88ca-4533c8297618: !Template + answer_choices: Yes ||| No + id: e83f2ca0-0c6a-4cf6-88ca-4533c8297618 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' diff --git a/promptsource/templates/blimp/regular_plural_subject_verb_agreement_2/templates.yaml b/promptsource/templates/blimp/regular_plural_subject_verb_agreement_2/templates.yaml new file mode 100644 index 000000000..2fde9c0ec --- /dev/null +++ b/promptsource/templates/blimp/regular_plural_subject_verb_agreement_2/templates.yaml @@ -0,0 +1,262 @@ +dataset: blimp +subset: regular_plural_subject_verb_agreement_2 +templates: + 3e972e06-8b0c-4a5b-8974-9ddff6808924: !Template + answer_choices: 1 ||| 2 + id: 3e972e06-8b0c-4a5b-8974-9ddff6808924 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_1_2 + reference: '' + 6e2d13c8-ad7b-409d-b32d-e8ae75ea7efa: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 6e2d13c8-ad7b-409d-b32d-e8ae75ea7efa + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + 7663aff0-04d8-4005-ba53-9b29f6bf0459: !Template + answer_choices: A ||| B + id: 7663aff0-04d8-4005-ba53-9b29f6bf0459 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_A_B + reference: '' + 7f72e931-7cc8-459c-8a72-f72da1f38165: !Template + answer_choices: 1 ||| 2 + id: 7f72e931-7cc8-459c-8a72-f72da1f38165 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_1_2 + reference: '' + 95d63d7e-0bb8-4d18-bd4e-b727a5ae1c8f: !Template + answer_choices: A ||| B + id: 95d63d7e-0bb8-4d18-bd4e-b727a5ae1c8f + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: better_A_B + reference: '' + ad7f9dd2-67ed-46cc-ad1e-f8c03bb2854d: !Template + answer_choices: Yes ||| No + id: ad7f9dd2-67ed-46cc-ad1e-f8c03bb2854d + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + c8735e72-1665-442d-9170-78c2fbf308e7: !Template + answer_choices: A ||| B + id: c8735e72-1665-442d-9170-78c2fbf308e7 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_A_B + reference: '' + f9d3b6b2-93ff-41d8-99c0-140462a8da47: !Template + answer_choices: Yes ||| No + id: f9d3b6b2-93ff-41d8-99c0-140462a8da47 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' diff --git a/promptsource/templates/blimp/sentential_negation_npi_licensor_present/templates.yaml b/promptsource/templates/blimp/sentential_negation_npi_licensor_present/templates.yaml new file mode 100644 index 000000000..1248c63a2 --- /dev/null +++ b/promptsource/templates/blimp/sentential_negation_npi_licensor_present/templates.yaml @@ -0,0 +1,262 @@ +dataset: blimp +subset: sentential_negation_npi_licensor_present +templates: + 20ea7037-f4bc-4d75-b51c-e76b702ab2af: !Template + answer_choices: A ||| B + id: 20ea7037-f4bc-4d75-b51c-e76b702ab2af + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_A_B + reference: '' + 3429788f-02cc-45ef-b33f-9c787470c350: !Template + answer_choices: Yes ||| No + id: 3429788f-02cc-45ef-b33f-9c787470c350 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 3589d43a-91b0-4e94-96d7-0b4ea7f2ae26: !Template + answer_choices: A ||| B + id: 3589d43a-91b0-4e94-96d7-0b4ea7f2ae26 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_A_B + reference: '' + 4a8cffd4-cfb2-45d4-af72-e4b47a8fa8b7: !Template + answer_choices: Yes ||| No + id: 4a8cffd4-cfb2-45d4-af72-e4b47a8fa8b7 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 67c9bf91-abfe-4d38-bab6-2b0f8f16833e: !Template + answer_choices: 1 ||| 2 + id: 67c9bf91-abfe-4d38-bab6-2b0f8f16833e + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_1_2 + reference: '' + 7091f7b6-b026-4f2d-a4d2-11da09bdb544: !Template + answer_choices: A ||| B + id: 7091f7b6-b026-4f2d-a4d2-11da09bdb544 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: better_A_B + reference: '' + 76c8e7a2-3746-4276-9b1e-ec34bf338bbc: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 76c8e7a2-3746-4276-9b1e-ec34bf338bbc + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + 9cd5bc3a-f7a7-4854-a32c-05f552b1e59b: !Template + answer_choices: 1 ||| 2 + id: 9cd5bc3a-f7a7-4854-a32c-05f552b1e59b + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_1_2 + reference: '' diff --git a/promptsource/templates/blimp/sentential_negation_npi_scope/templates.yaml b/promptsource/templates/blimp/sentential_negation_npi_scope/templates.yaml new file mode 100644 index 000000000..6c787077e --- /dev/null +++ b/promptsource/templates/blimp/sentential_negation_npi_scope/templates.yaml @@ -0,0 +1,262 @@ +dataset: blimp +subset: sentential_negation_npi_scope +templates: + 06bcd940-bd0b-469d-8270-5c10ce6724d6: !Template + answer_choices: 1 ||| 2 + id: 06bcd940-bd0b-469d-8270-5c10ce6724d6 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_1_2 + reference: '' + 24be7bee-d104-47bd-9dd0-dd6feaa5a50c: !Template + answer_choices: A ||| B + id: 24be7bee-d104-47bd-9dd0-dd6feaa5a50c + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_A_B + reference: '' + 29748319-4f33-48b9-94a3-b149623688af: !Template + answer_choices: Yes ||| No + id: 29748319-4f33-48b9-94a3-b149623688af + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 8077910e-4d3f-440f-aec8-cb8d5026add7: !Template + answer_choices: A ||| B + id: 8077910e-4d3f-440f-aec8-cb8d5026add7 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: better_A_B + reference: '' + ceaf4bf2-6a63-4d0f-88e5-da569756bfdc: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: ceaf4bf2-6a63-4d0f-88e5-da569756bfdc + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + d0b6a823-b8a8-4fc0-931c-4fdd59aa754f: !Template + answer_choices: Yes ||| No + id: d0b6a823-b8a8-4fc0-931c-4fdd59aa754f + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + d8a3adbb-8a01-424a-a20d-334b5649c20d: !Template + answer_choices: A ||| B + id: d8a3adbb-8a01-424a-a20d-334b5649c20d + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_A_B + reference: '' + e1b2a2c6-c9e8-4836-ab95-7607af2f0baa: !Template + answer_choices: 1 ||| 2 + id: e1b2a2c6-c9e8-4836-ab95-7607af2f0baa + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_1_2 + reference: '' diff --git a/promptsource/templates/blimp/sentential_subject_island/templates.yaml b/promptsource/templates/blimp/sentential_subject_island/templates.yaml new file mode 100644 index 000000000..150e21055 --- /dev/null +++ b/promptsource/templates/blimp/sentential_subject_island/templates.yaml @@ -0,0 +1,262 @@ +dataset: blimp +subset: sentential_subject_island +templates: + 33c5eb89-7d3f-4c06-b891-f76ebb029fa2: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 33c5eb89-7d3f-4c06-b891-f76ebb029fa2 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + 3bcabbab-d834-400b-8004-47c7af8d0296: !Template + answer_choices: Yes ||| No + id: 3bcabbab-d834-400b-8004-47c7af8d0296 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 5f943d95-000b-4d16-9de9-b41221af3f2a: !Template + answer_choices: Yes ||| No + id: 5f943d95-000b-4d16-9de9-b41221af3f2a + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 63fb3dc7-4361-4b92-806a-003bb95d3cbc: !Template + answer_choices: 1 ||| 2 + id: 63fb3dc7-4361-4b92-806a-003bb95d3cbc + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_1_2 + reference: '' + 8f8e59e9-9f15-45cc-bb1b-bb36f0b514bd: !Template + answer_choices: A ||| B + id: 8f8e59e9-9f15-45cc-bb1b-bb36f0b514bd + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: better_A_B + reference: '' + e7d92196-f32b-4922-a2b0-9d70977e1708: !Template + answer_choices: 1 ||| 2 + id: e7d92196-f32b-4922-a2b0-9d70977e1708 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_1_2 + reference: '' + f8a5abf1-e833-4095-b1c6-ed0f69f8353b: !Template + answer_choices: A ||| B + id: f8a5abf1-e833-4095-b1c6-ed0f69f8353b + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_A_B + reference: '' + f98877e8-23c3-4c3a-9747-c96b60444de2: !Template + answer_choices: A ||| B + id: f98877e8-23c3-4c3a-9747-c96b60444de2 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_A_B + reference: '' diff --git a/promptsource/templates/blimp/superlative_quantifiers_1/templates.yaml b/promptsource/templates/blimp/superlative_quantifiers_1/templates.yaml new file mode 100644 index 000000000..6a95b51d7 --- /dev/null +++ b/promptsource/templates/blimp/superlative_quantifiers_1/templates.yaml @@ -0,0 +1,262 @@ +dataset: blimp +subset: superlative_quantifiers_1 +templates: + 09ffbdea-f522-48ff-854a-c2b0a8864e46: !Template + answer_choices: 1 ||| 2 + id: 09ffbdea-f522-48ff-854a-c2b0a8864e46 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_1_2 + reference: '' + 2b5f03af-6ae6-4c97-bdf9-25bfde671223: !Template + answer_choices: A ||| B + id: 2b5f03af-6ae6-4c97-bdf9-25bfde671223 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_A_B + reference: '' + ac839ac0-5de1-44d8-8d8b-a0a074e4baac: !Template + answer_choices: A ||| B + id: ac839ac0-5de1-44d8-8d8b-a0a074e4baac + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_A_B + reference: '' + bd64feeb-3c96-441d-972f-36a6c3a2091d: !Template + answer_choices: 1 ||| 2 + id: bd64feeb-3c96-441d-972f-36a6c3a2091d + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_1_2 + reference: '' + bdc91436-3ced-43cb-984a-5964900cdde1: !Template + answer_choices: Yes ||| No + id: bdc91436-3ced-43cb-984a-5964900cdde1 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + cdbe007d-c8a1-4fa9-9fa0-1d171337c7d6: !Template + answer_choices: Yes ||| No + id: cdbe007d-c8a1-4fa9-9fa0-1d171337c7d6 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + d9516b74-8b12-4042-bec4-b8266c021428: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: d9516b74-8b12-4042-bec4-b8266c021428 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + f69f8b6d-2bfe-4e88-83a5-d38a46ef121c: !Template + answer_choices: A ||| B + id: f69f8b6d-2bfe-4e88-83a5-d38a46ef121c + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: better_A_B + reference: '' diff --git a/promptsource/templates/blimp/superlative_quantifiers_2/templates.yaml b/promptsource/templates/blimp/superlative_quantifiers_2/templates.yaml new file mode 100644 index 000000000..1eb53b205 --- /dev/null +++ b/promptsource/templates/blimp/superlative_quantifiers_2/templates.yaml @@ -0,0 +1,262 @@ +dataset: blimp +subset: superlative_quantifiers_2 +templates: + 28e41b28-a85d-4caf-b29b-2ff3aa909a42: !Template + answer_choices: A ||| B + id: 28e41b28-a85d-4caf-b29b-2ff3aa909a42 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: better_A_B + reference: '' + 39ef3b76-021e-48cd-83dc-00ff72f5a5e2: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 39ef3b76-021e-48cd-83dc-00ff72f5a5e2 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + 6bc37d1d-5868-4d84-92a1-0fdfada218d8: !Template + answer_choices: 1 ||| 2 + id: 6bc37d1d-5868-4d84-92a1-0fdfada218d8 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_1_2 + reference: '' + 860f5327-3af1-42dc-866f-dc310cf1b90f: !Template + answer_choices: A ||| B + id: 860f5327-3af1-42dc-866f-dc310cf1b90f + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_A_B + reference: '' + b8c6f707-38ef-464d-8df6-7132f159426e: !Template + answer_choices: Yes ||| No + id: b8c6f707-38ef-464d-8df6-7132f159426e + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + dc37ec11-c672-4fd7-a224-cb788e185ec6: !Template + answer_choices: Yes ||| No + id: dc37ec11-c672-4fd7-a224-cb788e185ec6 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + dc98c524-ca56-4650-b93e-38d2f142f000: !Template + answer_choices: A ||| B + id: dc98c524-ca56-4650-b93e-38d2f142f000 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_A_B + reference: '' + ec4db392-fe08-4c54-977f-fc53625374fb: !Template + answer_choices: 1 ||| 2 + id: ec4db392-fe08-4c54-977f-fc53625374fb + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_1_2 + reference: '' diff --git a/promptsource/templates/blimp/tough_vs_raising_1/templates.yaml b/promptsource/templates/blimp/tough_vs_raising_1/templates.yaml new file mode 100644 index 000000000..12f9c0da1 --- /dev/null +++ b/promptsource/templates/blimp/tough_vs_raising_1/templates.yaml @@ -0,0 +1,262 @@ +dataset: blimp +subset: tough_vs_raising_1 +templates: + 1a0874ea-9df5-4b93-a61d-2dedab8625bb: !Template + answer_choices: Yes ||| No + id: 1a0874ea-9df5-4b93-a61d-2dedab8625bb + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 1a96fecf-fb39-438a-b1dc-1dd0544203ac: !Template + answer_choices: 1 ||| 2 + id: 1a96fecf-fb39-438a-b1dc-1dd0544203ac + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_1_2 + reference: '' + 45b19636-cde2-4ce7-a036-d2b49608852c: !Template + answer_choices: Yes ||| No + id: 45b19636-cde2-4ce7-a036-d2b49608852c + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 5009927d-1b0d-4aba-89a2-3934e28c9989: !Template + answer_choices: A ||| B + id: 5009927d-1b0d-4aba-89a2-3934e28c9989 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_A_B + reference: '' + 6814638e-90c0-42fb-8cdc-5f0d3476bb74: !Template + answer_choices: 1 ||| 2 + id: 6814638e-90c0-42fb-8cdc-5f0d3476bb74 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_1_2 + reference: '' + baea91cd-56b7-4257-8590-39f468fe318d: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: baea91cd-56b7-4257-8590-39f468fe318d + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + d39f8df5-0280-4984-824e-776346bd0f12: !Template + answer_choices: A ||| B + id: d39f8df5-0280-4984-824e-776346bd0f12 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_A_B + reference: '' + f6046c53-3199-4e2d-82cc-e4dcfe733181: !Template + answer_choices: A ||| B + id: f6046c53-3199-4e2d-82cc-e4dcfe733181 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: better_A_B + reference: '' diff --git a/promptsource/templates/blimp/tough_vs_raising_2/templates.yaml b/promptsource/templates/blimp/tough_vs_raising_2/templates.yaml new file mode 100644 index 000000000..644536f34 --- /dev/null +++ b/promptsource/templates/blimp/tough_vs_raising_2/templates.yaml @@ -0,0 +1,262 @@ +dataset: blimp +subset: tough_vs_raising_2 +templates: + 31a0e88f-de15-450f-aa80-afc02eb1f6a7: !Template + answer_choices: A ||| B + id: 31a0e88f-de15-450f-aa80-afc02eb1f6a7 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_A_B + reference: '' + 42615eed-d287-47f6-88c8-f366ad6350ce: !Template + answer_choices: Yes ||| No + id: 42615eed-d287-47f6-88c8-f366ad6350ce + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 45838379-3e45-49cb-bb40-024291aa3ac7: !Template + answer_choices: Yes ||| No + id: 45838379-3e45-49cb-bb40-024291aa3ac7 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 77c255ac-eb29-4b5c-84ff-829ddd6d17ef: !Template + answer_choices: 1 ||| 2 + id: 77c255ac-eb29-4b5c-84ff-829ddd6d17ef + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_1_2 + reference: '' + b8acabf2-cc33-4283-a10e-b6d354d04a78: !Template + answer_choices: A ||| B + id: b8acabf2-cc33-4283-a10e-b6d354d04a78 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: better_A_B + reference: '' + d25542d3-3bad-4e3c-88bf-b513ece45759: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: d25542d3-3bad-4e3c-88bf-b513ece45759 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + e1cd87f2-cf74-481b-8fd2-2d5f24eb8585: !Template + answer_choices: 1 ||| 2 + id: e1cd87f2-cf74-481b-8fd2-2d5f24eb8585 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_1_2 + reference: '' + fd5ee495-2ddc-41cc-8e5d-d925de156a89: !Template + answer_choices: A ||| B + id: fd5ee495-2ddc-41cc-8e5d-d925de156a89 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_A_B + reference: '' diff --git a/promptsource/templates/blimp/transitive/templates.yaml b/promptsource/templates/blimp/transitive/templates.yaml new file mode 100644 index 000000000..c1e2f3487 --- /dev/null +++ b/promptsource/templates/blimp/transitive/templates.yaml @@ -0,0 +1,262 @@ +dataset: blimp +subset: transitive +templates: + 1c182220-eb99-4505-9d06-3cd032dee4d5: !Template + answer_choices: A ||| B + id: 1c182220-eb99-4505-9d06-3cd032dee4d5 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: better_A_B + reference: '' + 49c4bd4b-43aa-4020-8ec8-cc7a6644be19: !Template + answer_choices: 1 ||| 2 + id: 49c4bd4b-43aa-4020-8ec8-cc7a6644be19 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_1_2 + reference: '' + 4aed6b5d-b048-4715-8cac-3bc4b407e82d: !Template + answer_choices: A ||| B + id: 4aed6b5d-b048-4715-8cac-3bc4b407e82d + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_A_B + reference: '' + 4f7de6ef-e0fd-4d6d-8537-8b32af44a86d: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 4f7de6ef-e0fd-4d6d-8537-8b32af44a86d + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + 8c770286-45b2-461a-974b-48ef0cb7d0ac: !Template + answer_choices: Yes ||| No + id: 8c770286-45b2-461a-974b-48ef0cb7d0ac + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 9b8f0825-1565-439c-8f3d-3d620898e948: !Template + answer_choices: 1 ||| 2 + id: 9b8f0825-1565-439c-8f3d-3d620898e948 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_1_2 + reference: '' + b688ecf2-b95a-499f-b20f-c8a9e8305e22: !Template + answer_choices: A ||| B + id: b688ecf2-b95a-499f-b20f-c8a9e8305e22 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_A_B + reference: '' + f04f5a5b-5243-469b-8ff3-8edce3c4bcd8: !Template + answer_choices: Yes ||| No + id: f04f5a5b-5243-469b-8ff3-8edce3c4bcd8 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' diff --git a/promptsource/templates/blimp/wh_island/templates.yaml b/promptsource/templates/blimp/wh_island/templates.yaml new file mode 100644 index 000000000..e9aae6f22 --- /dev/null +++ b/promptsource/templates/blimp/wh_island/templates.yaml @@ -0,0 +1,262 @@ +dataset: blimp +subset: wh_island +templates: + 0be4ddb1-91d2-46e7-92ac-ba864eb83a6b: !Template + answer_choices: A ||| B + id: 0be4ddb1-91d2-46e7-92ac-ba864eb83a6b + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_A_B + reference: '' + 1354e34b-6ac5-4032-8cd0-329a76b24a11: !Template + answer_choices: Yes ||| No + id: 1354e34b-6ac5-4032-8cd0-329a76b24a11 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 81ca64d6-7bed-46ee-9c5a-b9c3395db11d: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 81ca64d6-7bed-46ee-9c5a-b9c3395db11d + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + 891da904-88cb-44c5-9a1a-90ad0842b316: !Template + answer_choices: A ||| B + id: 891da904-88cb-44c5-9a1a-90ad0842b316 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_A_B + reference: '' + 8a8d6781-67b5-46ed-8ebc-0af0e6cb304b: !Template + answer_choices: 1 ||| 2 + id: 8a8d6781-67b5-46ed-8ebc-0af0e6cb304b + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_1_2 + reference: '' + bbd8dbcb-7755-4444-bd40-2a9bdb2d5206: !Template + answer_choices: A ||| B + id: bbd8dbcb-7755-4444-bd40-2a9bdb2d5206 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: better_A_B + reference: '' + ebf77652-f993-43e0-b6b8-3c87e7b2a85e: !Template + answer_choices: 1 ||| 2 + id: ebf77652-f993-43e0-b6b8-3c87e7b2a85e + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_1_2 + reference: '' + ec1f1fb2-dacb-44f8-8870-5cb9f82b4ef2: !Template + answer_choices: Yes ||| No + id: ec1f1fb2-dacb-44f8-8870-5cb9f82b4ef2 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' diff --git a/promptsource/templates/blimp/wh_questions_object_gap/templates.yaml b/promptsource/templates/blimp/wh_questions_object_gap/templates.yaml new file mode 100644 index 000000000..b7958341d --- /dev/null +++ b/promptsource/templates/blimp/wh_questions_object_gap/templates.yaml @@ -0,0 +1,262 @@ +dataset: blimp +subset: wh_questions_object_gap +templates: + 40d31ad1-13e4-4ce3-b6eb-e1614ccaf11b: !Template + answer_choices: A ||| B + id: 40d31ad1-13e4-4ce3-b6eb-e1614ccaf11b + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_A_B + reference: '' + 536d5978-86d4-4146-a51f-e64cfbb8edc2: !Template + answer_choices: A ||| B + id: 536d5978-86d4-4146-a51f-e64cfbb8edc2 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_A_B + reference: '' + 560797ab-b625-4758-aec6-bde81ef69cb7: !Template + answer_choices: A ||| B + id: 560797ab-b625-4758-aec6-bde81ef69cb7 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: better_A_B + reference: '' + a2b50b51-922d-49a0-a84d-510665fb14d0: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: a2b50b51-922d-49a0-a84d-510665fb14d0 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + b54be2ff-4e50-4497-af1f-f7281db8aa77: !Template + answer_choices: 1 ||| 2 + id: b54be2ff-4e50-4497-af1f-f7281db8aa77 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_1_2 + reference: '' + ccfd0ff1-247e-4c39-9157-81275a5d5f77: !Template + answer_choices: Yes ||| No + id: ccfd0ff1-247e-4c39-9157-81275a5d5f77 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + e1219361-709d-467d-aeaf-5733e53e607e: !Template + answer_choices: Yes ||| No + id: e1219361-709d-467d-aeaf-5733e53e607e + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + ff4863c5-c8b4-476c-bc3c-a138bb226126: !Template + answer_choices: 1 ||| 2 + id: ff4863c5-c8b4-476c-bc3c-a138bb226126 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_1_2 + reference: '' diff --git a/promptsource/templates/blimp/wh_questions_subject_gap/templates.yaml b/promptsource/templates/blimp/wh_questions_subject_gap/templates.yaml new file mode 100644 index 000000000..56daffcba --- /dev/null +++ b/promptsource/templates/blimp/wh_questions_subject_gap/templates.yaml @@ -0,0 +1,262 @@ +dataset: blimp +subset: wh_questions_subject_gap +templates: + 04572431-8a02-4435-b06e-a1ffe47d1bc2: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 04572431-8a02-4435-b06e-a1ffe47d1bc2 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + 0bad60f2-5b52-4a99-a486-a4bcbd7839a5: !Template + answer_choices: 1 ||| 2 + id: 0bad60f2-5b52-4a99-a486-a4bcbd7839a5 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_1_2 + reference: '' + 0d64911f-41f6-4305-831b-f8e2e3474068: !Template + answer_choices: A ||| B + id: 0d64911f-41f6-4305-831b-f8e2e3474068 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_A_B + reference: '' + 1d281f37-5b87-43f9-94c3-bcc0e2d0f89d: !Template + answer_choices: A ||| B + id: 1d281f37-5b87-43f9-94c3-bcc0e2d0f89d + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_A_B + reference: '' + 781a0ca9-63ad-4d94-ae66-388246dabffe: !Template + answer_choices: A ||| B + id: 781a0ca9-63ad-4d94-ae66-388246dabffe + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: better_A_B + reference: '' + b2c97636-2bf7-429e-9af7-5186b5ea0e46: !Template + answer_choices: Yes ||| No + id: b2c97636-2bf7-429e-9af7-5186b5ea0e46 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + f4a1c41d-c3e1-4ddf-981c-cf4b8c8a9297: !Template + answer_choices: 1 ||| 2 + id: f4a1c41d-c3e1-4ddf-981c-cf4b8c8a9297 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_1_2 + reference: '' + f86d0901-0f03-452a-bfa6-b21e2b805171: !Template + answer_choices: Yes ||| No + id: f86d0901-0f03-452a-bfa6-b21e2b805171 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' diff --git a/promptsource/templates/blimp/wh_questions_subject_gap_long_distance/templates.yaml b/promptsource/templates/blimp/wh_questions_subject_gap_long_distance/templates.yaml new file mode 100644 index 000000000..3129d3b02 --- /dev/null +++ b/promptsource/templates/blimp/wh_questions_subject_gap_long_distance/templates.yaml @@ -0,0 +1,262 @@ +dataset: blimp +subset: wh_questions_subject_gap_long_distance +templates: + 29f05465-4289-4ded-9863-92e36d90c71f: !Template + answer_choices: Yes ||| No + id: 29f05465-4289-4ded-9863-92e36d90c71f + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 42eb0ad3-388d-43ad-af2c-a91e8b78db99: !Template + answer_choices: A ||| B + id: 42eb0ad3-388d-43ad-af2c-a91e8b78db99 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_A_B + reference: '' + 4330ea36-6c68-44cf-b91e-bb00b34bf1fa: !Template + answer_choices: Yes ||| No + id: 4330ea36-6c68-44cf-b91e-bb00b34bf1fa + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 6105db7f-bb5a-48fc-a8e2-0f5bc1034797: !Template + answer_choices: 1 ||| 2 + id: 6105db7f-bb5a-48fc-a8e2-0f5bc1034797 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_1_2 + reference: '' + 63bbab26-1eb1-4db0-bdb3-0c3b3e7a5ef9: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 63bbab26-1eb1-4db0-bdb3-0c3b3e7a5ef9 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + a281eb95-2ea6-4425-8344-d93598bfe1c7: !Template + answer_choices: A ||| B + id: a281eb95-2ea6-4425-8344-d93598bfe1c7 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_A_B + reference: '' + e8d445ca-9f07-48d4-8786-173bbf7319e1: !Template + answer_choices: 1 ||| 2 + id: e8d445ca-9f07-48d4-8786-173bbf7319e1 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_1_2 + reference: '' + ebb545b1-2e00-45a5-88ff-228d138cb157: !Template + answer_choices: A ||| B + id: ebb545b1-2e00-45a5-88ff-228d138cb157 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: better_A_B + reference: '' diff --git a/promptsource/templates/blimp/wh_vs_that_no_gap/templates.yaml b/promptsource/templates/blimp/wh_vs_that_no_gap/templates.yaml new file mode 100644 index 000000000..7f308b84a --- /dev/null +++ b/promptsource/templates/blimp/wh_vs_that_no_gap/templates.yaml @@ -0,0 +1,262 @@ +dataset: blimp +subset: wh_vs_that_no_gap +templates: + 1967f46d-e4fd-4cb6-94c6-692d6d7f0d95: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 1967f46d-e4fd-4cb6-94c6-692d6d7f0d95 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + 86dce9cf-2a28-41f8-8e37-0ed47dc8504e: !Template + answer_choices: 1 ||| 2 + id: 86dce9cf-2a28-41f8-8e37-0ed47dc8504e + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_1_2 + reference: '' + 8d29bf78-af7c-4b5f-b0a3-f001de1b17dc: !Template + answer_choices: A ||| B + id: 8d29bf78-af7c-4b5f-b0a3-f001de1b17dc + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: better_A_B + reference: '' + 92d37871-6182-4ae3-b853-2135d71c53c5: !Template + answer_choices: A ||| B + id: 92d37871-6182-4ae3-b853-2135d71c53c5 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_A_B + reference: '' + 968d8473-ffbe-4fc9-bcf4-c3f1cee02274: !Template + answer_choices: Yes ||| No + id: 968d8473-ffbe-4fc9-bcf4-c3f1cee02274 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 99b5cdd0-67b6-47ac-93d7-bc5dcd54c54b: !Template + answer_choices: Yes ||| No + id: 99b5cdd0-67b6-47ac-93d7-bc5dcd54c54b + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + dc7c7d5d-c532-4828-bb37-102bf1331b2f: !Template + answer_choices: 1 ||| 2 + id: dc7c7d5d-c532-4828-bb37-102bf1331b2f + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_1_2 + reference: '' + e61e4273-9bdd-42f8-9339-ce07df2feaee: !Template + answer_choices: A ||| B + id: e61e4273-9bdd-42f8-9339-ce07df2feaee + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_A_B + reference: '' diff --git a/promptsource/templates/blimp/wh_vs_that_no_gap_long_distance/templates.yaml b/promptsource/templates/blimp/wh_vs_that_no_gap_long_distance/templates.yaml new file mode 100644 index 000000000..a5ec43a5c --- /dev/null +++ b/promptsource/templates/blimp/wh_vs_that_no_gap_long_distance/templates.yaml @@ -0,0 +1,262 @@ +dataset: blimp +subset: wh_vs_that_no_gap_long_distance +templates: + 3b6b4263-da7a-4e05-a50b-ecdb4421196b: !Template + answer_choices: 1 ||| 2 + id: 3b6b4263-da7a-4e05-a50b-ecdb4421196b + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_1_2 + reference: '' + 4ee3c7ef-c1f3-4dd8-892d-d8d3d108f80c: !Template + answer_choices: Yes ||| No + id: 4ee3c7ef-c1f3-4dd8-892d-d8d3d108f80c + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 869e71d2-6236-4975-b6d1-838ef22a9dc4: !Template + answer_choices: A ||| B + id: 869e71d2-6236-4975-b6d1-838ef22a9dc4 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: better_A_B + reference: '' + 93962115-2842-47ad-bfbd-bfc8a54ebc99: !Template + answer_choices: A ||| B + id: 93962115-2842-47ad-bfbd-bfc8a54ebc99 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_A_B + reference: '' + 962c7019-9790-4204-86b5-bca38e94096c: !Template + answer_choices: 1 ||| 2 + id: 962c7019-9790-4204-86b5-bca38e94096c + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_1_2 + reference: '' + d241c093-8d87-451f-85fc-2baf3296fb2f: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: d241c093-8d87-451f-85fc-2baf3296fb2f + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + e7a4dc29-364a-48d3-9987-ee19a232e2df: !Template + answer_choices: Yes ||| No + id: e7a4dc29-364a-48d3-9987-ee19a232e2df + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + ee64e45c-4405-45bb-b7c2-e37ee785fd7b: !Template + answer_choices: A ||| B + id: ee64e45c-4405-45bb-b7c2-e37ee785fd7b + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_A_B + reference: '' diff --git a/promptsource/templates/blimp/wh_vs_that_with_gap/templates.yaml b/promptsource/templates/blimp/wh_vs_that_with_gap/templates.yaml new file mode 100644 index 000000000..ad2fe2b40 --- /dev/null +++ b/promptsource/templates/blimp/wh_vs_that_with_gap/templates.yaml @@ -0,0 +1,262 @@ +dataset: blimp +subset: wh_vs_that_with_gap +templates: + 36f1f042-b559-4787-8d68-0aa4e5817fee: !Template + answer_choices: Yes ||| No + id: 36f1f042-b559-4787-8d68-0aa4e5817fee + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 72645df4-fd34-4dcc-8bde-a51c1e9c1935: !Template + answer_choices: 1 ||| 2 + id: 72645df4-fd34-4dcc-8bde-a51c1e9c1935 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_1_2 + reference: '' + 8610967f-a5fd-463e-a9d7-a5c9811f5784: !Template + answer_choices: A ||| B + id: 8610967f-a5fd-463e-a9d7-a5c9811f5784 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_A_B + reference: '' + 8fa1171d-d5ec-479a-bd24-eb24150bdd4c: !Template + answer_choices: Yes ||| No + id: 8fa1171d-d5ec-479a-bd24-eb24150bdd4c + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + 93e1036a-f0f8-4fe0-804a-a3ba453879fb: !Template + answer_choices: A ||| B + id: 93e1036a-f0f8-4fe0-804a-a3ba453879fb + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_A_B + reference: '' + a4b730b4-dc8b-42c9-bb94-9bd679c7e326: !Template + answer_choices: A ||| B + id: a4b730b4-dc8b-42c9-bb94-9bd679c7e326 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: better_A_B + reference: '' + ab520b24-0b0c-4719-b2ac-b3e89d9357ae: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: ab520b24-0b0c-4719-b2ac-b3e89d9357ae + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + e7adcae8-837a-40eb-a97e-e4fb5b8f596c: !Template + answer_choices: 1 ||| 2 + id: e7adcae8-837a-40eb-a97e-e4fb5b8f596c + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_1_2 + reference: '' diff --git a/promptsource/templates/blimp/wh_vs_that_with_gap_long_distance/templates.yaml b/promptsource/templates/blimp/wh_vs_that_with_gap_long_distance/templates.yaml new file mode 100644 index 000000000..909435038 --- /dev/null +++ b/promptsource/templates/blimp/wh_vs_that_with_gap_long_distance/templates.yaml @@ -0,0 +1,262 @@ +dataset: blimp +subset: wh_vs_that_with_gap_long_distance +templates: + 070af6c5-71af-481a-aded-bd51f24ee3f8: !Template + answer_choices: A ||| B + id: 070af6c5-71af-481a-aded-bd51f24ee3f8 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_A_B + reference: '' + 0f27d869-9944-4d07-a461-709f991294f7: !Template + answer_choices: A ||| B + id: 0f27d869-9944-4d07-a461-709f991294f7 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Read the following two sentences. + + {% if shuffled_order == 0 %} + + Sentence A: {{ sentence_good }} + + Sentence B: {{ sentence_bad }} + + {% else %} + + Sentence A: {{ sentence_bad }} + + Sentence B: {{ sentence_good }} + + {% endif %} + + Which one is a better sentence of English, {% if shuffled_order_options == 0 + %}A or B{% else %}B or A{% endif %}? ||| {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: better_A_B + reference: '' + 16a913cc-56de-4785-acb1-e34c1f27c309: !Template + answer_choices: A ||| B + id: 16a913cc-56de-4785-acb1-e34c1f27c309 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}A or B{% else %}B or A{% endif %}. + + {% if shuffled_order == 0 %} + + A: {{ sentence_good }} + + B: {{ sentence_bad }} + + {% else %} + + A: {{ sentence_bad }} + + B: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_A_B + reference: '' + 4e312418-971d-405f-a25b-fd5b2f6c7444: !Template + answer_choices: '{{ sentence_good }} ||| {{sentence_bad}}' + id: 4e312418-971d-405f-a25b-fd5b2f6c7444 + jinja: ' ||| {{ sentence_good }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: null_prompt + reference: '' + 718c9c51-ee70-46bf-8663-0dfdcba56cae: !Template + answer_choices: Yes ||| No + id: 718c9c51-ee70-46bf-8663-0dfdcba56cae + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a good sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[0] }} {% else %} {{ answer_choices[1] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_good_yes_no + reference: '' + 87a19169-f4df-4c72-a1cd-2b394a7fffda: !Template + answer_choices: 1 ||| 2 + id: 87a19169-f4df-4c72-a1cd-2b394a7fffda + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Between the two sentences, which one is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_between_1_2 + reference: '' + 8878b790-491e-47da-aa7b-b216ec80223a: !Template + answer_choices: Yes ||| No + id: 8878b790-491e-47da-aa7b-b216ec80223a + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + "{% if shuffled_order == 0 %}{{ sentence_good }}{% else %}{{ sentence_bad }}{% + endif %}" + + Is this sentence a bad sentence of English, {% if shuffled_order_options == + 0 %}Yes or No{% else %}No or Yes{% endif %}? ||| {% if shuffled_order == 0 %} + {{ answer_choices[1] }} {% else %} {{ answer_choices[0] }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: single_sentence_bad_yes_no + reference: '' + fea9a9cc-dc36-48de-92a4-e81bc3dd7220: !Template + answer_choices: 1 ||| 2 + id: fea9a9cc-dc36-48de-92a4-e81bc3dd7220 + jinja: '{% set shuffled_order = [0, 1] | random %} + + {% set shuffled_order_options = [0, 1] | random %} + + Which one of the following sentences is grammatical? Please answer {% if shuffled_order_options + == 0 %}1 or 2{% else %}2 or 1{% endif %}. + + {% if shuffled_order == 0 %} + + 1: {{ sentence_good }} + + 2: {{ sentence_bad }} + + {% else %} + + 1: {{ sentence_bad }} + + 2: {{ sentence_good }} + + {% endif %} + + ||| + + {% if shuffled_order == 0 %} + + {{ answer_choices[0] }} + + {% else %} + + {{ answer_choices[1] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: grammatical_which_one_1_2 + reference: ''