From 55bd4513d5a1c1ec6060ee820d2fa1c293d8c118 Mon Sep 17 00:00:00 2001 From: Daniel Drabik Date: Wed, 8 Jan 2025 14:53:35 -0500 Subject: [PATCH 01/51] wip --- .../gen_ai/evaluation/prompt_builder.rb | 29 +++++++++++++++ .../evaluation/prompts/2025_01_08_initial.md | 37 +++++++++++++++++++ 2 files changed, 66 insertions(+) create mode 100644 services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/prompt_builder.rb create mode 100644 services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/prompts/2025_01_08_initial.md diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/prompt_builder.rb b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/prompt_builder.rb new file mode 100644 index 00000000000..f4c2451e470 --- /dev/null +++ b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/prompt_builder.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Evidence + module GenAI + module Evaluation + class PromptBuilder < ApplicationService + TEMPLATE_FOLDER = 'app/services/evidence/gen_ai/evaluation/prompts/' + TEMPLATE = '2025_01_08_initial.md' + + attr_reader :entry + + def initialize(entry:) + @entry = entry + end + + private def template_variables + { + entry:, + } + end + + def run = template % template_variables + + private def template = File.read(template_file_path) + private def template_file_path = Evidence::Engine.root.join(TEMPLATE_FOLDER, TEMPLATE) + end + end + end +end diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/prompts/2025_01_08_initial.md b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/prompts/2025_01_08_initial.md new file mode 100644 index 00000000000..f5ac895d0e4 --- /dev/null +++ b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/prompts/2025_01_08_initial.md @@ -0,0 +1,37 @@ +You are an expert feedback evaluator. You will analyze a student's answer and a piece of feedback given and determine if it meets the feedback guidelines. + +Answer +``` +%s +``` + +Go through these rules one by one and evaluate the feedback. If the feedback does not meet the criteria of a rule, return JSON {"optimal" : false, "rule" : "rule_name"} where the "rule_name" is the name of the rule. Stop evaluating rule if you hit a failing rule. If the feedback passes all of the rules, return {"optimal" : true} + +### Rules +|rule|criteria| +|----|----| +|grammar| Do not give grammar, spelling, or punction feedback.| +|be_concise| Do not tell the student to 'Be Concise' or something similar| +|mixed_message| Do not give conflicting advise, e.g. "good work, now clear your response.", "that's true, but is also not true"| +|not_in_student_text| If the feedback "Yes, and" the student's answer, it must only use the details in the answer: '%s'. | +|two_directions| Only give one instruction for the next step. Acknowledgement of what the student wrote, e.g. "That's true that pizza is delicious" do not count as instructions. Note, one instruction can be written in two sentences if it is the same thought, e.g. "Now add detail about why the student body protested. Why did they walk out?" would only be one instruction| +|gives_answer| Only discuss facts in the student's answer, do not give away specific facts or details for the student to use except for acknowledging the text in their answer. Instead, You should redirect them to the text to find a detail or ask them a question about the text| + + +### Examples +|answer|feedback|optimal|rule| +|----|----|----|----| +|I don't understand the question| Try clearing your response and starting again. What requirement kept some boys from volunteering in the British Army during World War I? Check that your response only uses information from the text.|true|| +|some boys really wanted to enlist| Instead of focusing on a reason why some boys wanted to volunteer during WWI, write about what prevented them from being able to enlist. What prevented some boys from volunteering in the British Army?|true|| +|it's over their|That's true. Now correct the spelling of the word 'Their'|false|grammar| +|ocean pollution is bad|That's true. Add some detail to your answer about ocean pollution. Also, list companies that are pollutors|false|two_directions| +|ocean pollution is bad|That's true that pollution is bad. Why is pollution bad?|true|| +|the pizza is hot|Nice work, you are on the right track. Clear your response and start again with details from the passage|false|mixed_message| +|because blue whales are the largest mammals.|That's true, blue whales are the largest mammals. How large can they grow to?|true|| +|because blue whales eat plankton.|You're on the right track, blue whales are the largest mammals. How large can they grow to?|false|not_in_student_text| +|pizza is delicous.|That's true, pizza is made with cheese. What other ingredients are used?|false|not_in_student_text| +|because blue whales are the largest mammals|You're on the right track. The text also states that Blue whales can weigh 100 tons. Include that in your response|false|gives_answer| +|I tried and tried and tried and tried.|That's good, now make your answer a little shorter and less wordy|false|be_concise| + + + From c391c58ebab37e3428b613f5b208c6298ad3c750 Mon Sep 17 00:00:00 2001 From: Daniel Drabik Date: Fri, 10 Jan 2025 11:07:06 -0500 Subject: [PATCH 02/51] Wip --- .../evaluation/data/whales_suboptimal.csv | 188 ++++++++++++++++++ .../gen_ai/evaluation/data_fetcher.rb | 34 ++++ .../gen_ai/evaluation/prompt_builder.rb | 14 ++ .../evaluation/prompts/2025_01_08_initial.md | 14 +- .../QuillLMS/lib/tasks/gen_ai_evaluation.thor | 82 ++++++++ 5 files changed, 321 insertions(+), 11 deletions(-) create mode 100644 services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/data/whales_suboptimal.csv create mode 100644 services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/data_fetcher.rb create mode 100644 services/QuillLMS/lib/tasks/gen_ai_evaluation.thor diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/data/whales_suboptimal.csv b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/data/whales_suboptimal.csv new file mode 100644 index 00000000000..29de0a06fe7 --- /dev/null +++ b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/data/whales_suboptimal.csv @@ -0,0 +1,188 @@ +entry,feedback,optimal_broad,optimal,rule, +they sending drones to gather listening devices to the whale.,"It's true that CETI is using drones. Now be more specific. Why are they sending drones, and what are they gathering?",FALSE,FALSE,two_directions, +they use drones to attach specially-designed listening devices.,It's true that they're using drones and specially-designed listening devices! Now add to your response. How are they using the drones and listening devices to collect more data?,FALSE,TRUE,unhelpful, +the CETI team is working on solutions like creating drones specifically designed to listen to whales.,It's true that CETI is using drones. Now add to your response. What are the drones doing and how are they helping CETI get the data they need?,FALSE,FALSE,two_directions, +they can more effectively find the patterns in the communications of whales.,"Try clearing your response and trying again. Instead of explaining how the data will be used, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +the complexities of the whale clicks can be unraveled.,"Try clearing your response and trying again. Instead of talking about the complexities of whale clicks, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +they have made drones that have attatched specially designed listening devices.,It's true that they're using drones with listening devices! Now add to your response. How are they attaching the listening devices to collect the whale sounds?,TRUE,TRUE,, +they can talk to whales.,"Try clearing your response and trying again. Instead of talking about CETI's mission to communicate with whales, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +researchers hope that the project will help humans engage in dialogue with whales.,"Try clearing your response and trying again. Instead of talking about CETI's mission to communicate with whales, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +"Eventually, it will be possible to decipher the whales' language and converse with them.","Try clearing your response and trying again. Instead of talking about CETI's mission to communicate with whales, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +scientists and researchers can learn more quickly in research.,"Try clearing your response and trying again. Instead of talking about the benefits of collecting more data, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +scientist can study whale communication.,"Try clearing your response and trying again. Instead of talking about what scientists can do with the data, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +"in the future, decoding the whales' language for communication with the creatures is a potential development.","Try clearing your response and trying again. Instead of talking about CETI's mission to communicate with whales, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +humans can engage with species at risk of extinction.,"Try clearing your response and trying again. Instead of talking about CETI's mission to communicate with whales, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +when there on the move they can communicate with the whales,"Try clearing your response and trying again. Instead of talking about how whales communicate, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",FALSE,FALSE,misacknowledgement, +"at present, they possess only thousands of whale recordings, which falls short of what's required to comprehend a language.","Try clearing your response and trying again. Instead of explaining how much data is needed, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +"currently, they have only thousands of whale recordings, which is insufficient for understanding a language.","Try clearing your response and trying again. Instead of explaining how much data is needed, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +People now have a greater understanding of animals.,"Try clearing your response and trying again. Instead of talking about what humans might learn from whales, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +eventually the whales' language can be decoded and used to communicate with the creatures.,"Try clearing your response and trying again. Instead of talking about CETI's mission to communicate with whales, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +they can be able to collect recordings of whales clicking.,"Try clearing your response and trying again. Instead of talking about what CETI can do with the data, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +they use whales and their language.,"Try clearing your response and trying again. Instead of talking about whales and their language, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +"what they are missing are recordings of tens of millions of whales, which they need.","Try clearing your response and trying again. Instead of explaining how much data is needed, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +they can be able to collect data and communicate with whales.,"Try clearing your response and trying again. Instead of talking about CETI's mission to communicate with whales, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +they can hear the voices of these whales that are thousands of feet deep in the water.,"Try clearing your response and trying again. Instead of talking about what CETI can do with the data, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +they can easily identify on what the whales are saying.,"Try clearing your response and trying again. Instead of talking about CETI's mission to communicate with whales, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +they will have to catch a whale because they are always on the move.,"Try clearing your response and trying again. Instead of talking about the challenges of catching whales, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +"presently, they hold only thousands of whale recordings, and this quantity is insufficient for understanding a language.","Try clearing your response and trying again. Instead of explaining how much data is needed, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +efforts to prevent extinction can be expanded through learning as much as possible about whales.,"Try clearing your response and trying again. Instead of talking about CETI's mission to communicate with whales, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +they plan to use LLM to firgure out how whales communicate.,"Try clearing your response and trying again. Instead of talking about the methods CETI is using to understand whale communication, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",FALSE,FALSE,mixed_message, +researchers can effectively translate different whale vibrations and sounds.,"Try clearing your response and trying again. Instead of talking about CETI's mission to communicate with whales, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +"what's missing from their collection are recordings in the tens of millions of whales, which they need.","Try clearing your response and trying again. Instead of explaining how much data is needed, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +"humans will be able to talk to whales, and whales will be able to talk with humans.","Try clearing your response and trying again. Instead of talking about CETI's mission to communicate with whales, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +"their present holdings number in the thousands, but they require tens of millions to meet their needs.","Try clearing your response and trying again. Instead of explaining how much data is needed, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +there will be a clear line of communication.,"Try clearing your response and trying again. Instead of talking about CETI's mission to communicate with whales, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +"what they require are recordings in the tens of millions of whales, which they currently do not have.","Try clearing your response and trying again. Instead of explaining how much data is needed, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +the long-term goal is to decode the language of the whales and use it for communication with the creatures.,"Try clearing your response and trying again. Instead of talking about CETI's mission to communicate with whales, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +"they require whale recordings in the tens of millions, which they do not currently have.","Try clearing your response and trying again. Instead of explaining how much data is needed, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +"their current count is in the thousands, which is far from the tens of millions they require.","Try clearing your response and trying again. Instead of explaining how much data is needed, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +"right now, they are in possession of only thousands of whale recordings, and this number is insufficient for language understanding.","Try clearing your response and trying again. Instead of explaining how much data is needed, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +It is possible to decipher the complexity of the whale clicks.,"Try clearing your response and trying again. Instead of talking about what is possible to do with the data, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +they can communicate with whales better without any misunderstandings.,"Try clearing your response and trying again. Instead of talking about CETI's mission to communicate with whales, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +"they have thus far accumulated only thousands, whereas tens of millions are still required.","Try clearing your response and trying again. Instead of explaining how much data is needed, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +"in the future, the whales' language could be deciphered and utilized to interact with these creatures.","Try clearing your response and trying again. Instead of talking about CETI's mission to communicate with whales, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +they need thousands of more sightings of it.,"Try clearing your response and trying again. Instead of explaining how much data is needed, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +"they currently possess only thousands of whale recordings, which proves inadequate for language understanding.","Try clearing your response and trying again. Instead of explaining how much data is needed, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +"Hopefully, the team will be able to keep coming up with new methods to further its use in learning about whales.","Try clearing your response and trying again. Instead of talking about the future of CETI's research, focus on how CETI is getting their data now. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +they have collected thousands of whale recordings to see a pattern in whale communication.,"Try clearing your response and trying again. Instead of explaining how much data CETI has, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +they only have thousands out of the tens of millions of data points they need.,"Try clearing your response and trying again. Instead of explaining how much data is needed, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +"team has thousands of whale recordings, but to find the patterns in their communication,","Try clearing your response and trying again. Instead of explaining how much data is needed, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +"through the analysis of numerous language samples, an LLM can find patterns that underlie the language's operation.","Try clearing your response and trying again. Instead of explaining how LLMs work, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +"by reviewing a diverse set of language instances, an LLM can map out the rules and patterns it discovers.","Try clearing your response and trying again. Instead of explaining how LLMs work, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +the CETI team has thousands of whale recordings.,"Try clearing your response and trying again. Instead of explaining how much data CETI has, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +they have millions of recordings from when they looked for whales.,"Try clearing your response and trying again. Instead of explaining how much data is needed, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +in order to find a pattern in the recordings they would need to have tens of millions of recordings instead of the thousands.,"Try clearing your response and trying again. Instead of explaining how much data is needed, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +lLMs can enhance their word comprehension in specific contexts through the utilization of a massive dataset.,"Try clearing your response and trying again. Instead of explaining how LLMs work, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +they are often looking at the patterns of whale communication and has over 10 million recordings off of the whales that they have recording devices on.,"Try clearing your response and trying again. Instead of explaining how much data is needed, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +"with access to copious data, lLMs can refine their understanding of a word in specific contexts.","Try clearing your response and trying again. Instead of explaining how LLMs use data, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +AI will work correctly.,"Try clearing your response and trying again. Instead of talking about what AI will do, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +they have thousands of recordings.,"Try clearing your response and trying again. Instead of explaining how much data CETI has, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +they might need tens of millions of recordings to translate the whale clicks.,"Try clearing your response and trying again. Instead of explaining how much data is needed, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +"through the examination of a vast number of language examples, an LLM can uncover patterns governing the language's structure.","Try clearing your response and trying again. Instead of explaining how LLMs work, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +"by harnessing a huge dataset, lLMs can better grasp a word in specific contexts with increased accuracy.","Try clearing your response and trying again. Instead of explaining how LLMs work, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +it does not lead to misunderstanding and false conclusions.,"Try clearing your response and trying again. Instead of explaining a reason CETI needs large amounts of data, focus on how they are getting this data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +misunderstandings and false conclusions can be avoided.,"Try clearing your response and trying again. Instead of explaining a reason CETI needs large amounts of data, focus on how they are getting this data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +they can accurately train the models and won't make mistakes.,"Try clearing your response and trying again. Instead of explaining why CETI needs large amounts of data, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +there are no false conclusions.,"Try clearing your response and trying again. Instead of explaining a reason CETI needs large amounts of data, focus on how they are getting this data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +"The CETI team has thousands of whale recordings, but to find the patterns in their communication, they would likely need tens of millions of recordings.","Try clearing your response and trying again. Instead of explaining how much data is needed, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +data sets that are too small can lead to misunderstandings or false conclusions.,"Try clearing your response and trying again. Instead of explaining a reason CETI needs large amounts of data, focus on how they are getting this data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +a small dataset can lead to misunderstandings and false conclusions.,"Try clearing your response and trying again. Instead of explaining a reason CETI needs large amounts of data, focus on how they are getting this data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +a small dataset could result in misunderstandings and wrong conclusions.,"Try clearing your response and trying again. Instead of explaining a reason CETI needs large amounts of data, focus on how they are getting this data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +they work by analyzing a massive dataset and finding the patterns in it,"Try clearing your response and trying again. Instead of explaining how LLMs work, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +they're now using specially attached drones made just for listening.,"It's true that CETI is using drones. Now add to your response. What are the drones attaching to, and why?",TRUE,TRUE,, +a vast amount of data can assist lLMs in comprehending a word with greater precision in particular contexts.,"Try clearing your response and trying again. Instead of explaining why large amounts of data are needed, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +there are not any misunderstandings and confusions.,"Try clearing your response and trying again. Instead of explaining a reason CETI needs large amounts of data, focus on how they are getting this data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +if it is too small it can lead to misunderstandings and wrong info.,"Try clearing your response and trying again. Instead of explaining a reason CETI needs large amounts of data, focus on how they are getting this data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +small amounts of data could cause misunderstandings.,"Try clearing your response and trying again. Instead of explaining a reason CETI needs large amounts of data, focus on how they are getting this data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +It is unclear what message or information is being conveyed in this text.,"Try clearing your response and trying again. Instead of summarizing the text, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +a complete understanding of them necessitates listening to them in their own setting.,"Try clearing your response and trying again. Instead of talking about the importance of listening to whales, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +they assign a series of numbers to each small piece of language.,"Try clearing your response and trying again. Instead of explaining how LLMs process data, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +a small amount of data may produce wrong results and create misunderstandings.,"Try clearing your response and trying again. Instead of explaining a reason CETI needs large amounts of data, focus on how they are getting this data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +they can find the rules and patterns by looking at many examples.,"Try clearing your response and trying again. Instead of explaining how LLMs work, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +whales are helping.,"Try clearing your response and trying again. Instead of talking about whales helping, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +making whale click language is very difficult.,"Try clearing your response and trying again. Instead of talking about the difficulty of whale communication, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +anything smaller can lead to misunderstandings and false conclusions.,"Try clearing your response and trying again. Instead of explaining a reason CETI needs large amounts of data, focus on how they are getting this data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +a dataset that is too small could lead to misunderstandings and false conclusions.,"Try clearing your response and trying again. Instead of explaining a reason CETI needs large amounts of data, focus on how they are getting this data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +life is sweet like cinammon.,"Try clearing your response and trying again. Instead of talking about a general topic, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",FALSE,TRUE,bonkers, +This input does not represent a meaningful sentence.,"Try clearing your response and trying again. Instead of explaining how much data is needed, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",FALSE,FALSE,misacknowledgement, +"if they use a small dataset, it could cause the researchers to misunderstand the meaning of whale codas.","Try clearing your response and trying again. Instead of explaining a reason CETI needs large amounts of data, focus on how they are getting this data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +it can cost a lot o,"Try clearing your response and trying again. Instead of talking about the cost of collecting data, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",FALSE,TRUE,bonkers, +they can analyze patterns and map out rules for the language by creating large datasets.,"Try clearing your response and trying again. Instead of explaining how LLMs work, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +they're going to,"Try clearing your response and trying again. Instead of talking about what CETI is going to do, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",FALSE,TRUE,bonkers,Match +the more modals the better.,"Try clearing your response and trying again. Instead of talking about the number of modals, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",FALSE,TRUE,bonkers, +they are trying to narrow the amount of formation used to make it more efficient.,"Try clearing your response and trying again. Instead of talking about making the data more efficient, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",FALSE,TRUE,bonkers, +"CETI requires extensive date sets for the successful training of its expansive language models, because large-scale training date can promote the machin","Try clearing your response and trying again. Instead of explaining why large datasets are needed, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +they find it difficult to understand what the whales say.,"Try clearing your response and trying again. Instead of talking about the difficulty of understanding whales, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +whales are intellingent they converse with other whales sjdd,"Try clearing your response and trying again. Instead of talking about whales' intelligence, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +they need to learn easily,"Try clearing your response and trying again. Instead of explaining how LLMs learn, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",FALSE,TRUE,bonkers, +ta da da,"Try clearing your response and trying again. Instead of writing something unrelated, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",FALSE,TRUE,tone, +tey need beter partssss,"Try clearing your response and trying again. Instead of explaining how much data is needed, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",FALSE,TRUE,bonkers, +it is super difficult.,"Try clearing your response and trying again. Instead of explaining the difficulty of collecting data, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +they're attaching drones to whales.,"It's true that CETI is using drones. Now add to your response. What are they attaching to the whales with the drones, and why?",FALSE,FALSE,two_directions, +they use suction cups.,It's true that CETI is using suction cups! Now add to your response. What are they using suction cups to attach?,TRUE,TRUE,, +they use a type of suction cup.,It's true that CETI is using suction cups. Now add to your response. What are they using suction cups to attach?,TRUE,TRUE,, +they are using underwater microphones.,It's true that CETI is using underwater microphones. Now be more specific. How are they using these microphones to collect data?,TRUE,TRUE,, +they came up with unique solutions such as suction cup devices.,"It's true that CETI is using suction cups. Now add to your response. What are they using suction cups to attach, and why?",FALSE,FALSE,two_directions, +they find whales and use specific tools like suction cups.,"It's true that they are finding whales and using suction cups. Now add to your response. What are they using the suction cups to attach, and why are they doing this?",FALSE,FALSE,two_directions, +they attach many suction cups.,It's true that they're using suction cups! Now add to your response. What are they using suction cups to attach?,TRUE,TRUE,, +they are using suction cups.,It's true that CETI is using suction cups. Now add to your response. What are they using suction cups to attach?,TRUE,TRUE,, +"they are using drones, and suction cups.",It's true that CETI is using drones and suction cups! Now add to your response. What are they using the drones and suction cups to attach?,TRUE,TRUE,, +they put suction cups on the whales.,It's true that they're using suction cups! Now add to your response. What are they using suction cups to attach?,TRUE,TRUE,, +they will attach to the whale with a suction cup.,It's true that they are using suction cups! Now add to your response. What are they attaching to the whales with the suction cups?,TRUE,TRUE,, +they attach suction cups to the whale.,It's true that they're using suction cups! Now add to your response. What are they using suction cups to attach?,TRUE,TRUE,, +the team is working on unique solutions to get it.,It's true that CETI is working on unique solutions to get more data. Now be more specific. What kinds of solutions are they using?,TRUE,TRUE,, +they use many different devices to record the sound.,"It's true that CETI is using many different devices. Now be more specific. What kinds of devices are they using, and how are they using them?",FALSE,FALSE,two_directions, +they are utilizing audio recording devices,"It's true that CETI is using audio recording devices. Now be more specific. What kinds of devices are they using, and how are they using them to collect data?",FALSE,FALSE,two_directions, +the team uses some unique methods.,It's true that CETI is using unique methods. Now be more specific. What methods are they using?,TRUE,TRUE,, +they use listening devices to gather data.,It's true that CETI is using listening devices to get more data. Now be more specific. What are the listening devices and how are they using them?,FALSE,FALSE,two_directions, +the CETI team are using devices to listen to whales.,"It's true that CETI is using devices to listen to whales. Now add to your response. What kinds of devices are they using, and how are they using them?",FALSE,FALSE,two_directions, +they are using drones and suction cup.,It's true that CETI is using drones and suction cups! Now add to your response. What are they using the suction cups to attach?,TRUE,TRUE,, +they have unique listening devices.,It's true that CETI is using unique listening devices. Now be more specific. What are these devices and how are they using them?,FALSE,FALSE,two_directions, +the team is finding unique ways to get the data.,You're on the right track. Now add to your response. What unique ways are they using to get the data?,TRUE,TRUE,, +it is planning to get more data.,"Try clearing your response and trying again. Instead of explaining that CETI is planning to get more data, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +they send out many recording devices,"Try clearing your response and trying again. Instead of explaining how CETI is getting data, focus on what types of recording devices they are using and how they are using them. What is the CETI team doing to get the recordings they need?",FALSE,FALSE,two_directions, +Scholars have devised many methods for gathering click recordings.,"Try clearing your response and trying again. Instead of explaining the methods used, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",FALSE,TRUE,, +scientist can use special hearing devices to hear what the whales are saying.,"Try clearing your response and trying again. Instead of talking about CETI's mission to communicate with whales, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",FALSE,FALSE,misacknowledgement, +"To get it, the team is developing original solutions.",It's true that CETI is developing original solutions. Now be more specific. What kinds of solutions are they using?,TRUE,TRUE,, +they have to work hard to find it.,"Try clearing your response and trying again. Instead of explaining the difficulty of collecting data, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",FALSE,FALSE,misacknowledgement, +The CETI team is developing a number of tools to make data collecting easier.,It's true that CETI is working on unique solutions to get more data. Now be more specific. What kinds of solutions are they using?,TRUE,TRUE,, +It intends to obtain further data.,"Try clearing your response and trying again. Instead of explaining that CETI intends to obtain further data, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",FALSE,FALSE,mixed_message, +teams are doing more recordings.,"Try clearing your response and trying again. Instead of explaining how much data is needed, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",FALSE,FALSE,misacknowledgement, +They are developing novel methods to collect data on whales.,It's true that CETI is developing novel methods. Now be more specific. What methods are they using?,TRUE,TRUE,, +They are developing some original solutions.,It's true that CETI is working on unique solutions to get more data. Now be more specific. What kinds of solutions are they using?,TRUE,TRUE,, +Groups are recording more content.,"Try clearing your response and trying again. Instead of explaining how much data is needed, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",FALSE,FALSE,misacknowledgement, +The group is coming up with inventive methods to obtain the information.,"Try clearing your response and trying again. Instead of talking about CETI's methods in general terms, focus on the specific methods they are using to collect whale sounds. What are they doing to get the recordings they need?",FALSE,FALSE,mixed_message, +they have begun to use suction cups.,"It's true that CETI is using suction cups. Now add to your response. What are they using suction cups to attach, and why?",FALSE,FALSE,two_directions, +It wants to gather more information.,"Try clearing your response and trying again. Instead of explaining why CETI needs large amounts of data, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +CETI is working on creating some unique methods to efficiently gather this data.,It's true that CETI is working on unique solutions to get more data. Now be more specific. What kinds of solutions are they using?,TRUE,TRUE,, +they're recruiting the assistance of a drone.,It's true that CETI is using drones. Now add to your response. What are they using the drones for?,TRUE,TRUE,, +"in order to get the data that they need, the CETI team is working on some unique solutions.",It's true that CETI is working on unique solutions to get more data. Now be more specific. What kinds of solutions are they using?,TRUE,TRUE,, +they are recruiting the assistance of a drone.,It's true that CETI is using drones. Now add to your response. What are they using the drones for?,TRUE,TRUE,, +scientists use an abundance of tools to study whale communication.,"Try clearing your response and trying again. Instead of talking about the tools used to study whale communication, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",FALSE,FALSE,mixed_message, +"the CETI team has come up with several unique solutions, like drones.","It's true that CETI is using drones. Now add to your response. What are they using the drones for, and what other solutions are they using?",FALSE,FALSE,two_directions, +they spent most of their times working on unique solutions.,"Try clearing your response and trying again. Instead of explaining what the CETI team is doing, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",FALSE,TRUE,bonkers, +the CETI team is creating unique solutions.,It's true that CETI is working on unique solutions to get more data. Now be more specific. What kinds of solutions are they using?,TRUE,TRUE,, +"the CETI group has devised a variety of distinctive alternatives, such as a drone.","It's true that CETI is using drones. Now add to your response. What are they using the drones for, and what other methods are they using?",FALSE,FALSE,two_directions, +CETI is working on some unique solutions.,It's true that CETI is working on unique solutions to get more data. Now be more specific. What kinds of solutions are they using?,TRUE,TRUE,, +"the CETI group has devised numerous distinctive alternatives, such as a drone.","It's true that CETI is using drones. Now add to your response. What are they using the drones for, and how does that help them get more data?",FALSE,FALSE,two_directions, +how they're getting their data they're studying the whales to see what will help them.,"Try clearing your response and trying again. Instead of talking about studying whales, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",FALSE,TRUE,, +"Innovative remedies, like a drone, are being implemented.","It's true that CETI is using drones. Now add to your response. What are they using the drones for, and what else are they doing to collect data?",FALSE,FALSE,two_directions, +the CETI team is working on unique solution for the problem.,It's true that CETI is working on unique solutions to get more data. Now be more specific. What kinds of solutions are they using?,TRUE,TRUE,, +"the CETI team must collect a large amount of data by tracking and sighting whales to help their dataset be more effective, even though it is incredibly hard to get data on whales.","Try clearing your response and trying again. Instead of explaining the challenges of collecting whale data, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +they are utilizing a drone.,"It's true that CETI is using drones. Now add to your response. Why are they using drones, and what do the drones do to help them collect data?",FALSE,FALSE,two_directions, +the CETI team is working on a better way to catch whale dataset.,"Try clearing your response and trying again. Instead of explaining how CETI is working on a better way, focus on what methods they are using to collect data. What is the CETI team doing to get the recordings they need?",FALSE,FALSE,mixed_message, +often they send researchers to go collect data.,"Try clearing your response and trying again. Instead of talking about how researchers collect data, focus on what methods CETI is using to get the recordings they need. What specific technologies or strategies are they employing?",TRUE,TRUE,, +they spend weeks at a time trying to collect that data,"Try clearing your response and trying again. Instead of explaining how difficult it is to collect data, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, +when they use drones they can get it back.,"Try clearing your response and trying again. Instead of explaining how the drones help, focus on what the drones are used for. What is the CETI team doing to get the recordings they need?",FALSE,FALSE,mixed_message, +CETI is constantly coming up with unique solutions to train their LLMs.,"Try clearing your response and trying again. Instead of talking about the solutions CETI is coming up with, focus on what those solutions are. What is the CETI team doing to get the recordings they need?",FALSE,FALSE,mixed_message, +they are finding ways to get more of it.,"Try clearing your response and trying again. Instead of explaining how CETI is getting more data, focus on what methods they are using. What specific actions is the CETI team taking to get the recordings they need?",FALSE,FALSE,mixed_message, +the CETI team is working on unique solutiond such as using drones.,It's true that CETI is working on unique solutions to get more data. Now be more specific. What kinds of solutions are they using?,FALSE,FALSE,gives_answer, +the CETI team is collecting date from various sources.,"Try clearing your response and trying again. Instead of explaining how CETI is getting data, focus on the specific methods they are using. What is the CETI team doing to get the recordings they need?",FALSE,FALSE,mixed_message, +they're engaging the aid of a drone.,It's true that CETI is using drones. Now add to your response. What are they using the drones for?,TRUE,TRUE,, +devices have been made to collect data.,"Try clearing your response and trying again. Instead of explaining what devices are used, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",FALSE,FALSE,mixed_message, +the CETI team is working on some unique solutions for example they are using drone technology.,"It's true that CETI is working on unique solutions and using drone technology. Now add to your response. What are they using the drones for, and what other solutions are they using?",FALSE,FALSE,two_directions, +they use drones and microphones.,It's true that CETI is using drones and microphones. Now add to your response. How are they using these tools to collect more data?,FALSE,FALSE,gives_answer, +they are working on unique solutions by using drones.,"It's true that CETI is working on unique solutions and using drones. Now add to your response. What are they using the drones for, and what other solutions are they using?",FALSE,FALSE,two_directions, +they are using drones to attach to whales.,"It's true that CETI is using drones. Now add to your response. What are they attaching to the whales with the drones, and why are they doing this?",FALSE,FALSE,two_directions, +they are utilizing a drone and acoustic sensors.,It's true that CETI is using drones and acoustic sensors. Now add to your response. How are they using these tools to collect more data?,FALSE,FALSE,gives_answer, +they are utilizing mics thousands of feet beneath the sea.,It's true that CETI is placing listening devices on the seafloor. Now be more specific. Why are they doing this?,TRUE,TRUE,, +"the CETI group is developing one-of-a-kind remedies, including specialized audio equipment on the seaflooor.",It's true that CETI is working on unique solutions to get more data. Now be more specific. What kinds of solutions are they using?,FALSE,TRUE,, +"the CETI team is developing distinctive remedies, such as specialized eavesdropping equipment thousands of feet below.","Try clearing your response and trying again. Instead of talking about the equipment, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",FALSE,FALSE,mixed_message, +the CETI team is working on some unique solutions like listening devices on the ocean floor.,It's true that CETI is placing listening devices on the ocean floor. Now be more specific. Why are they doing this?,TRUE,TRUE,, +they are utilizing a drone and audio recording devices.,It's true that CETI is using drones and audio recording devices. Now add to your response. How are they using these tools to collect more data?,FALSE,FALSE,gives_answer, +CETI is putting listening devices on the seafloor.,It's true that CETI is placing listening devices on the seafloor. Now be more specific. Why are they doing this?,TRUE,TRUE,, +they're putting underwater microphones on the sea floor.,It's true that CETI is placing listening devices on the seafloor. Now be more specific. Why are they doing this?,TRUE,TRUE,, +they're using underwater mics on the bottom of the ocean.,It's true that CETI is placing listening devices on the seafloor. Now be more specific. Why are they doing this?,TRUE,TRUE,, +they're using listening devices that go at the bottom of the ocean.,It's true that CETI is placing listening devices on the seafloor. Now be more specific. Why are they doing this?,TRUE,TRUE,, +CETI is placing listening devices on the bottom of the ocean.,It's true that CETI is placing listening devices on the seafloor. Now be more specific. Why are they doing this?,TRUE,TRUE,, +they use mics that work thousands of feet underwater.,It's true that CETI is using underwater microphones. Now add to your response. How are they using these microphones to collect data?,FALSE,FALSE,gives_answer, +they are employing a drone and audio recorders.,It's true that CETI is using drones and audio recorders. Now add to your response. How are they using the drones and audio recorders to collect more data?,FALSE,FALSE,gives_answer, +they're recording at the bottom of the seafloor.,It's true that CETI is placing listening devices on the seafloor. Now be more specific. Why are they doing this?,TRUE,TRUE,, +they are also placing listening devices on the sea floor.,It's true that CETI is placing listening devices on the seafloor. Now be more specific. Why are they doing this?,TRUE,TRUE,, +they're placing special listening devises on the bottom of the ocean.,It's true that CETI is placing listening devices on the seafloor. Now be more specific. Why are they doing this?,TRUE,TRUE,, +the CETI group is currently developing distinct remedies such as specialized hearing equipment on the bottom of the ocean.,It's true that CETI is working on unique solutions to get more data. Now be more specific. What kinds of solutions are they using?,FALSE,TRUE,, +they're using drones to attach to whales.,"It's true that CETI is using drones. Now add to your response. What are they attaching to the whales, and why?",FALSE,FALSE,two_directions, +Scientists affix microphones to whales.,"It's true that scientists are using microphones. Now add to your response. How are they affixing the microphones to the whales, and what is the purpose of doing so?",FALSE,FALSE,two_directions, +they are placing devices to capture whale clicks.,It's true that CETI is placing devices to capture whale clicks. Now be more specific. What kind of devices are they using and where are they placing them?,FALSE,FALSE,two_directions, +they will attach listening devices to the whales.,It's true that CETI is attaching listening devices. Now add to your response. How are they attaching them and for how long will the devices record?,FALSE,FALSE,two_directions, +scientist have created a device that attaches to the whales.,"It's true that scientists created a device that attaches to whales. Now add to your response. What is the purpose of this device, and how does it help CETI collect data?",FALSE,FALSE,two_directions, \ No newline at end of file diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/data_fetcher.rb b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/data_fetcher.rb new file mode 100644 index 00000000000..309287b3e8c --- /dev/null +++ b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/data_fetcher.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module Evidence + module GenAI + module Evaluation + class DataFetcher < ApplicationService + Item = Data.define(:entry, :feedback, :optimal_broad, :optimal, :rule) + + DEFAULT_FILE = 'whales_suboptimal.csv' + CSV_FILE_PATH = "#{Evidence::Engine.root}/app/services/evidence/gen_ai/evaluation/data/%s" + + def run = data + + private def data + @data ||= csv_data.map { |row| item_from_row(row) } + end + + private def csv_data = CSV.read(file_path, headers: true) + private def file_path = format(CSV_FILE_PATH, file:) + private def file = DEFAULT_FILE + + private def item_from_row(row) + Item.new( + entry: row['entry'], + feedback: row['feedback'], + optimal_broad: ActiveModel::Type::Boolean.new.cast(row['optimal_broad']), + optimal: ActiveModel::Type::Boolean.new.cast(row['optimal']), + rule: row['rule'] + ) + end + end + end + end +end diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/prompt_builder.rb b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/prompt_builder.rb index f4c2451e470..e33d49e18bd 100644 --- a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/prompt_builder.rb +++ b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/prompt_builder.rb @@ -4,9 +4,20 @@ module Evidence module GenAI module Evaluation class PromptBuilder < ApplicationService + include Evidence::MarkdownFormatter + TEMPLATE_FOLDER = 'app/services/evidence/gen_ai/evaluation/prompts/' TEMPLATE = '2025_01_08_initial.md' + RULES = { + grammar: 'Do not give grammar, spelling, or punctuation feedback.', + be_concise: "Do not tell the student to 'Be Concise' or something similar.", + mixed_message: 'Do not give conflicting advise, e.g. "good work, now clear your response.", "that\'s true, but is also not true"', + misacknowledgement: 'If the feedback "Yes, and" the student\'s answer, it must only use the details in the student\'s answer.', + two_directions: 'Only give one instruction for the next step. Acknowledgements of what the student wrote, e.g. "That\'s true that pizza is delicious" do not count as instructions. Note, one instruction can be written in two sentences if it is the same thought, e.g. "Now add detail about why the student body protested. Why did they walk out?" would only be one instruction', + gives_answer: "Only discuss facts in the student's answer, do not give away specific facts or details for the student to use except for acknowledging the text in their answer. Instead, You should redirect them to the text to find a detail or ask them a question about the text", + } + attr_reader :entry def initialize(entry:) @@ -16,11 +27,14 @@ def initialize(entry:) private def template_variables { entry:, + rules_table: } end def run = template % template_variables + private def rules_table = markdown_table_rows(rules) + private def rules = RULES.stringify_keys.to_a private def template = File.read(template_file_path) private def template_file_path = Evidence::Engine.root.join(TEMPLATE_FOLDER, TEMPLATE) end diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/prompts/2025_01_08_initial.md b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/prompts/2025_01_08_initial.md index f5ac895d0e4..30b94b02ea3 100644 --- a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/prompts/2025_01_08_initial.md +++ b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/prompts/2025_01_08_initial.md @@ -10,12 +10,7 @@ Go through these rules one by one and evaluate the feedback. If the feedback doe ### Rules |rule|criteria| |----|----| -|grammar| Do not give grammar, spelling, or punction feedback.| -|be_concise| Do not tell the student to 'Be Concise' or something similar| -|mixed_message| Do not give conflicting advise, e.g. "good work, now clear your response.", "that's true, but is also not true"| -|not_in_student_text| If the feedback "Yes, and" the student's answer, it must only use the details in the answer: '%s'. | -|two_directions| Only give one instruction for the next step. Acknowledgement of what the student wrote, e.g. "That's true that pizza is delicious" do not count as instructions. Note, one instruction can be written in two sentences if it is the same thought, e.g. "Now add detail about why the student body protested. Why did they walk out?" would only be one instruction| -|gives_answer| Only discuss facts in the student's answer, do not give away specific facts or details for the student to use except for acknowledging the text in their answer. Instead, You should redirect them to the text to find a detail or ask them a question about the text| +%s ### Examples @@ -28,10 +23,7 @@ Go through these rules one by one and evaluate the feedback. If the feedback doe |ocean pollution is bad|That's true that pollution is bad. Why is pollution bad?|true|| |the pizza is hot|Nice work, you are on the right track. Clear your response and start again with details from the passage|false|mixed_message| |because blue whales are the largest mammals.|That's true, blue whales are the largest mammals. How large can they grow to?|true|| -|because blue whales eat plankton.|You're on the right track, blue whales are the largest mammals. How large can they grow to?|false|not_in_student_text| -|pizza is delicous.|That's true, pizza is made with cheese. What other ingredients are used?|false|not_in_student_text| +|because blue whales eat plankton.|You're on the right track, blue whales are the largest mammals. How large can they grow to?|false|misacknowledgement| +|pizza is delicous.|That's true, pizza is made with cheese. What other ingredients are used?|false|misacknowledgement| |because blue whales are the largest mammals|You're on the right track. The text also states that Blue whales can weigh 100 tons. Include that in your response|false|gives_answer| |I tried and tried and tried and tried.|That's good, now make your answer a little shorter and less wordy|false|be_concise| - - - diff --git a/services/QuillLMS/lib/tasks/gen_ai_evaluation.thor b/services/QuillLMS/lib/tasks/gen_ai_evaluation.thor new file mode 100644 index 00000000000..407213133d5 --- /dev/null +++ b/services/QuillLMS/lib/tasks/gen_ai_evaluation.thor @@ -0,0 +1,82 @@ +# frozen_string_literal: true + +require_relative '../../config/environment' + +class GenAIEvaluation < Thor + Result = Data.define(:entry, :feedback, :optimal, :optimal_output, :optimal_correct, :rule, :rule_output) + + # bundle exec thor gen_a_i_evaluation:run_trial 5 + desc 'run_trial 5', 'Run an trial of off-topic llm prompt' + def run_trial(limit = 200) + results = [] + + test_data(limit).each do |item| + + entry = item.entry + optimal = item.optimal + feedback = item.feedback + rule = item.rule + system_prompt = system_prompt(entry) + + response = Evidence::Gemini::Chat.run(system_prompt:, entry: feedback, model: 'gemini-1.5-pro-latest') + + optimal_output = response['optimal'] + optimal_correct = optimal_output == optimal + rule_output = response['rule'] + + results << Result.new(entry:, feedback:, optimal:, optimal_output:, optimal_correct:, rule:, rule_output:) + end + + print_results(results) + + save_csv(results, limit) + end + + # put helper methods in this block + no_commands do + GEN_AI_OUTPUT_FOLDER = ENV.fetch('GEN_AI_OUTPUT_FOLDER', Rails.root.join('/lib/data/')) + + private def output_file(limit) + "#{GEN_AI_OUTPUT_FOLDER}_evalution_#{limit}_#{Time.now.to_i}.csv" + end + + private def save_csv(results, limit) + CSV.open(output_file(limit), 'wb') do |csv| + csv << Result.members.map(&:to_s) + results.each { |data| csv << data.deconstruct } + end + end + + private def print_results(results) + total = results.size + correct = results.count(&:optimal_correct) + + optimal_results = results.select(&:optimal) + optimal_total = optimal_results.count + optimal_correct = optimal_results.count(&:optimal_correct) + + suboptimal_results = results.reject(&:optimal) + suboptimal_total = suboptimal_results.count + suboptimal_correct = suboptimal_results.count(&:optimal_correct) + + puts 'Results' + print_line + print_result('Total', correct, total) + print_result('Optimal', optimal_correct, optimal_total) + print_result('Suboptimal', suboptimal_correct, suboptimal_total) + end + + private def print_result(name, numerator, denominator) + puts "#{name}: #{numerator}/#{denominator} #{(numerator / denominator.to_f).round(2)}" + print_line + end + + private def test_data(limit) = Evidence::GenAI::Evaluation::DataFetcher.run.first(limit.to_i) + + private def system_prompt(entry) = Evidence::GenAI::Evaluation::PromptBuilder.run(entry:) + + private def print_line + puts '---------------' + end + end +end From b16de96fc8dccf2c0d22265f3ea9934e53308876 Mon Sep 17 00:00:00 2001 From: Daniel Drabik Date: Fri, 10 Jan 2025 15:20:13 -0500 Subject: [PATCH 03/51] Working version of rule trial. --- .../gen_ai/evaluation/data_fetcher.rb | 18 +++++- .../gen_ai/evaluation/prompt_builder.rb | 19 +++---- .../prompts/2025_01_10_only_feedback.md | 24 ++++++++ .../QuillLMS/lib/tasks/gen_ai_evaluation.thor | 56 ++++++++++++------- 4 files changed, 83 insertions(+), 34 deletions(-) create mode 100644 services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/prompts/2025_01_10_only_feedback.md diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/data_fetcher.rb b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/data_fetcher.rb index 309287b3e8c..03e4246cf6b 100644 --- a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/data_fetcher.rb +++ b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/data_fetcher.rb @@ -4,12 +4,25 @@ module Evidence module GenAI module Evaluation class DataFetcher < ApplicationService - Item = Data.define(:entry, :feedback, :optimal_broad, :optimal, :rule) + Item = Data.define(:feedback, :optimal_broad, :optimal, :rule) DEFAULT_FILE = 'whales_suboptimal.csv' CSV_FILE_PATH = "#{Evidence::Engine.root}/app/services/evidence/gen_ai/evaluation/data/%s" - def run = data + ALL_RULES = Evidence::GenAI::Evaluation::PromptBuilder + + attr_reader :rule, :optimal + + def initialize(rule: nil, optimal: nil) + @rule = rule + @optimal = optimal + end + + def run + data + .select { |i| rule.nil? ? true : i.rule == rule } + .select { |i| optimal.nil? ? true : i.optimal == optimal } + end private def data @data ||= csv_data.map { |row| item_from_row(row) } @@ -21,7 +34,6 @@ def run = data private def item_from_row(row) Item.new( - entry: row['entry'], feedback: row['feedback'], optimal_broad: ActiveModel::Type::Boolean.new.cast(row['optimal_broad']), optimal: ActiveModel::Type::Boolean.new.cast(row['optimal']), diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/prompt_builder.rb b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/prompt_builder.rb index e33d49e18bd..58255777281 100644 --- a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/prompt_builder.rb +++ b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/prompt_builder.rb @@ -7,26 +7,25 @@ class PromptBuilder < ApplicationService include Evidence::MarkdownFormatter TEMPLATE_FOLDER = 'app/services/evidence/gen_ai/evaluation/prompts/' - TEMPLATE = '2025_01_08_initial.md' + TEMPLATE = '2025_01_10_only_feedback.md' RULES = { + verbose: 'Do not write more than 4 sentences.', grammar: 'Do not give grammar, spelling, or punctuation feedback.', + two_question: 'Do not ask more that one question. Even one question with two parts is not allowed', + repeats_question: 'Do not repeat the question or talk about the conjunction, e.g. "The sentence says x, so do this", "The conjunction \'but\' was used..."', be_concise: "Do not tell the student to 'Be Concise' or something similar.", mixed_message: 'Do not give conflicting advise, e.g. "good work, now clear your response.", "that\'s true, but is also not true"', - misacknowledgement: 'If the feedback "Yes, and" the student\'s answer, it must only use the details in the student\'s answer.', - two_directions: 'Only give one instruction for the next step. Acknowledgements of what the student wrote, e.g. "That\'s true that pizza is delicious" do not count as instructions. Note, one instruction can be written in two sentences if it is the same thought, e.g. "Now add detail about why the student body protested. Why did they walk out?" would only be one instruction', - gives_answer: "Only discuss facts in the student's answer, do not give away specific facts or details for the student to use except for acknowledging the text in their answer. Instead, You should redirect them to the text to find a detail or ask them a question about the text", + two_directions: "Do not give more than one direction, even in a single sentence joined by an 'and'. These structures are both wrong: \"Do this. Also do this.\", \"Do this, and do this.\"" } - attr_reader :entry - - def initialize(entry:) - @entry = entry - end + UNUSED_RULES = { + gives_answer: "Only discuss facts in the student's answer, do not give away specific facts or details for the student to use except for acknowledging the text in their answer. Instead, You should redirect them to the text to find a detail or ask them a question about the text", + misacknowledgement: 'If the feedback "Yes, and" the student\'s answer, it must only use the details in the student\'s answer.', + } private def template_variables { - entry:, rules_table: } end diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/prompts/2025_01_10_only_feedback.md b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/prompts/2025_01_10_only_feedback.md new file mode 100644 index 00000000000..0e0e51b0b68 --- /dev/null +++ b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/prompts/2025_01_10_only_feedback.md @@ -0,0 +1,24 @@ +You are an expert feedback evaluator. You will analyze a piece of feedback given and determine if it meets the rules. + +Go through these rules one by one and evaluate the feedback. If the feedback does not meet the criteria of a rule, return JSON {"optimal" : false, "rule" : "rule_name"} where the "rule_name" is the name of the rule. Stop evaluating rule if you hit a failing rule. If the feedback passes all of the rules, return {"optimal" : true} + +### Rules +|rule|criteria| +|----|----| +%s + + +### Examples +|feedback|optimal|rule| +|----|----|----|----| +|Try clearing your response and starting again. What requirement kept some boys from volunteering in the British Army during World War I? Check that your response only uses information from the text.|true|| +|Instead of focusing on a reason why some boys wanted to volunteer during WWI, write about what prevented them from being able to enlist. What prevented some boys from volunteering in the British Army?|true|| +|That's true. Now correct the spelling of the word 'Their'|false|grammar| +|That's true. Add some detail to your answer about ocean pollution. Also, list companies that are pollutors|false|two_directions| +|That's true that pollution is bad. Why is pollution bad?|true|| +|Nice work, you are on the right track. Clear your response and start again with details from the passage|false|mixed_message| +|That's true, blue whales are the largest mammals. How large can they grow to?|true|| +|That's good, now make your answer a little shorter and less wordy|false|be_concise| +|That's true. Add some detail to your answer about ocean pollution. Also, list companies that are pollutors|false|two_directions| +|That's true. Why do you think the people revolted? What else may be a factor?|false|two_questions| +|That's true, many underage boys did not last long in the war. However, the sentence starts with 'but,' which indicates a contrasting idea. The question asks what prevented many young teenage boys from volunteering. What was the age requirement for the British Army?|false|repeats_question| diff --git a/services/QuillLMS/lib/tasks/gen_ai_evaluation.thor b/services/QuillLMS/lib/tasks/gen_ai_evaluation.thor index 407213133d5..915dd6d1aae 100644 --- a/services/QuillLMS/lib/tasks/gen_ai_evaluation.thor +++ b/services/QuillLMS/lib/tasks/gen_ai_evaluation.thor @@ -3,39 +3,53 @@ require_relative '../../config/environment' class GenAIEvaluation < Thor - Result = Data.define(:entry, :feedback, :optimal, :optimal_output, :optimal_correct, :rule, :rule_output) + Result = Data.define(:feedback, :optimal, :optimal_output, :optimal_correct, :rule, :rule_output) # bundle exec thor gen_a_i_evaluation:run_trial 5 - desc 'run_trial 5', 'Run an trial of off-topic llm prompt' + desc 'run_trial 5', 'Run an feedback evalution trial' def run_trial(limit = 200) - results = [] + data = test_data(limit) - test_data(limit).each do |item| - - entry = item.entry - optimal = item.optimal - feedback = item.feedback - rule = item.rule - system_prompt = system_prompt(entry) - - response = Evidence::Gemini::Chat.run(system_prompt:, entry: feedback, model: 'gemini-1.5-pro-latest') + run_test(data, limit) + end - optimal_output = response['optimal'] - optimal_correct = optimal_output == optimal - rule_output = response['rule'] + # bundle exec thor gen_a_i_evaluation:run_rule rule 5 + desc 'run_rule rule limit', 'Run an feedback evalution trial for a rule' + def run_rule(rule, limit = 100) + optimal_data = Evidence::GenAI::Evaluation::DataFetcher.run(optimal: true).first(limit.to_i) + suboptimal_data = Evidence::GenAI::Evaluation::DataFetcher.run(optimal: false, rule:).first(limit.to_i) - results << Result.new(entry:, feedback:, optimal:, optimal_output:, optimal_correct:, rule:, rule_output:) - end + data = [optimal_data, suboptimal_data].flatten - print_results(results) - - save_csv(results, limit) + run_test(data, limit) end # put helper methods in this block no_commands do GEN_AI_OUTPUT_FOLDER = ENV.fetch('GEN_AI_OUTPUT_FOLDER', Rails.root.join('/lib/data/')) + private def run_test(data, limit) + results = [] + + data.each do |item| + optimal = item.optimal + feedback = item.feedback + rule = item.rule + + response = Evidence::Gemini::Chat.run(system_prompt:, entry: feedback, model: 'gemini-1.5-pro-latest') + + optimal_output = response['optimal'] + optimal_correct = optimal_output == optimal + rule_output = response['rule'] + + results << Result.new(feedback:, optimal:, optimal_output:, optimal_correct:, rule:, rule_output:) + end + + print_results(results) + + save_csv(results, limit) + end + private def output_file(limit) "#{GEN_AI_OUTPUT_FOLDER}_evalution_#{limit}_#{Time.now.to_i}.csv" end @@ -73,7 +87,7 @@ class GenAIEvaluation < Thor private def test_data(limit) = Evidence::GenAI::Evaluation::DataFetcher.run.first(limit.to_i) - private def system_prompt(entry) = Evidence::GenAI::Evaluation::PromptBuilder.run(entry:) + private def system_prompt = Evidence::GenAI::Evaluation::PromptBuilder.run private def print_line puts '---------------' From 58c4c2f5ae127c5d9fda122fac25319759501194 Mon Sep 17 00:00:00 2001 From: Daniel Drabik Date: Fri, 10 Jan 2025 16:41:35 -0500 Subject: [PATCH 04/51] Extract examples out of markdown and into code to make easier to edit. --- .../gen_ai/evaluation/prompt_builder.rb | 55 +++++++++++++++++-- .../prompts/2025_01_10_only_feedback.md | 14 +---- 2 files changed, 53 insertions(+), 16 deletions(-) diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/prompt_builder.rb b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/prompt_builder.rb index 58255777281..bd40040d464 100644 --- a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/prompt_builder.rb +++ b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/prompt_builder.rb @@ -10,13 +10,13 @@ class PromptBuilder < ApplicationService TEMPLATE = '2025_01_10_only_feedback.md' RULES = { - verbose: 'Do not write more than 4 sentences.', + # verbose: 'Do not write more than 4 sentences.', grammar: 'Do not give grammar, spelling, or punctuation feedback.', - two_question: 'Do not ask more that one question. Even one question with two parts is not allowed', + two_questions: 'Do not ask more that one question. Even one question with two parts is not allowed', repeats_question: 'Do not repeat the question or talk about the conjunction, e.g. "The sentence says x, so do this", "The conjunction \'but\' was used..."', be_concise: "Do not tell the student to 'Be Concise' or something similar.", mixed_message: 'Do not give conflicting advise, e.g. "good work, now clear your response.", "that\'s true, but is also not true"', - two_directions: "Do not give more than one direction, even in a single sentence joined by an 'and'. These structures are both wrong: \"Do this. Also do this.\", \"Do this, and do this.\"" + two_directions: "Ignore question sentences and count the number of next steps given by the feedback. That number should only be one." } UNUSED_RULES = { @@ -24,16 +24,63 @@ class PromptBuilder < ApplicationService misacknowledgement: 'If the feedback "Yes, and" the student\'s answer, it must only use the details in the student\'s answer.', } + SUBOPTIMAL_EXAMPLES = { + grammar: [ + "That's true. Now correct the spelling of the word 'Their'", + "Nice work. Your answer needs an apostrophe to show possesion, please add." + ], + two_questions: [ + "That's true. Why do you think the people revolted? What else may be a factor?" + ], + repeats_question: [ + "That's true, many underage boys did not last long in the war. However, the sentence starts with 'but,' which indicates a contrasting idea. The question asks what prevented many young teenage boys from volunteering. What was the age requirement for the British Army?" + ], + be_concise: [ + "That's good, now make your answer a little shorter and less wordy." + ], + mixed_message: [ + "Nice work, you are on the right track. Clear your response and start again with details from the passage" + ], + two_directions: [ + "That's true. Add some detail to your answer about ocean pollution. Also, list companies that are pollutors." + ] + } + + OPTIMAL_EXAMPLES = [ + "It's true that some boys were too young to enlist. Add another detail to make your response more specific. What was the age requirement for soldiers?", + "It's true that many boys did not meet the age requirement. Add another detail to make your response more specific. What was the age requirement for soldiers?", + "Try clearing your response and starting again. What requirement kept some boys from volunteering in the British Army during World War I? Check that your response only uses information from the text.", + "Instead of focusing on a reason why some boys wanted to volunteer during WWI, write about what prevented them from being able to enlist. What prevented some boys from volunteering in the British Army?", + "Try clearing your response and starting again. Instead of talking about how TrailGuard computer vision cameras need radio signal or internet service to work, talk about the cameras' technical limitations. What technical limitation can TrailGuard cameras have?", + "That's true that pollution is bad. What causes pollution to be bad?", + "That's true, blue whales are the largest mammals. How large can they grow to be?", + "That's true! Now add more specific information to your response. What is one main cause of the elephant poaching problem?", + "That's true - the TrailGuard computer vision system can occasionally lose connectivity! Now add a specific detail to your response. Where can TrailGuard cameras occasionally lose connection?" + ] + private def template_variables { - rules_table: + rules_table:, + examples_table: } end def run = template % template_variables + + private def examples_table = markdown_table_rows(examples) + private def examples = optimal_rows + suboptimal_rows + private def optimal_rows = OPTIMAL_EXAMPLES.map{|e| [e,true,'']} + private def suboptimal_rows + SUBOPTIMAL_EXAMPLES + .to_a + .map {|rule, array| array.map{|f| [f, false, rule.to_s]}} + .flatten(1) + end + private def rules_table = markdown_table_rows(rules) private def rules = RULES.stringify_keys.to_a + private def template = File.read(template_file_path) private def template_file_path = Evidence::Engine.root.join(TEMPLATE_FOLDER, TEMPLATE) end diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/prompts/2025_01_10_only_feedback.md b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/prompts/2025_01_10_only_feedback.md index 0e0e51b0b68..186e462df65 100644 --- a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/prompts/2025_01_10_only_feedback.md +++ b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/prompts/2025_01_10_only_feedback.md @@ -10,15 +10,5 @@ Go through these rules one by one and evaluate the feedback. If the feedback doe ### Examples |feedback|optimal|rule| -|----|----|----|----| -|Try clearing your response and starting again. What requirement kept some boys from volunteering in the British Army during World War I? Check that your response only uses information from the text.|true|| -|Instead of focusing on a reason why some boys wanted to volunteer during WWI, write about what prevented them from being able to enlist. What prevented some boys from volunteering in the British Army?|true|| -|That's true. Now correct the spelling of the word 'Their'|false|grammar| -|That's true. Add some detail to your answer about ocean pollution. Also, list companies that are pollutors|false|two_directions| -|That's true that pollution is bad. Why is pollution bad?|true|| -|Nice work, you are on the right track. Clear your response and start again with details from the passage|false|mixed_message| -|That's true, blue whales are the largest mammals. How large can they grow to?|true|| -|That's good, now make your answer a little shorter and less wordy|false|be_concise| -|That's true. Add some detail to your answer about ocean pollution. Also, list companies that are pollutors|false|two_directions| -|That's true. Why do you think the people revolted? What else may be a factor?|false|two_questions| -|That's true, many underage boys did not last long in the war. However, the sentence starts with 'but,' which indicates a contrasting idea. The question asks what prevented many young teenage boys from volunteering. What was the age requirement for the British Army?|false|repeats_question| +|----|----|----| +%s From 4d6ab0af5696256454c1e1d8adfcf58fc9c6098a Mon Sep 17 00:00:00 2001 From: Daniel Drabik Date: Mon, 13 Jan 2025 16:47:54 -0500 Subject: [PATCH 05/51] Add String extensions to count questions. --- .../evidence/lib/core_ext/extensions.rb | 1 + .../lib/core_ext/string_extensions.rb | 24 +++++++++++++++++++ .../engines/evidence/lib/evidence/engine.rb | 5 ++++ 3 files changed, 30 insertions(+) create mode 100644 services/QuillLMS/engines/evidence/lib/core_ext/extensions.rb create mode 100644 services/QuillLMS/engines/evidence/lib/core_ext/string_extensions.rb diff --git a/services/QuillLMS/engines/evidence/lib/core_ext/extensions.rb b/services/QuillLMS/engines/evidence/lib/core_ext/extensions.rb new file mode 100644 index 00000000000..e619160ec9d --- /dev/null +++ b/services/QuillLMS/engines/evidence/lib/core_ext/extensions.rb @@ -0,0 +1 @@ +require_relative 'string_extensions' diff --git a/services/QuillLMS/engines/evidence/lib/core_ext/string_extensions.rb b/services/QuillLMS/engines/evidence/lib/core_ext/string_extensions.rb new file mode 100644 index 00000000000..2f43b8b81a1 --- /dev/null +++ b/services/QuillLMS/engines/evidence/lib/core_ext/string_extensions.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +class String + SENTENCE_PATTERN = /(? Date: Mon, 13 Jan 2025 16:48:16 -0500 Subject: [PATCH 06/51] WIP for regex version of prompt checker. --- .../evidence/gen_ai/evaluation/checker.rb | 34 +++++++++++++++ .../evaluation/direction_count/checker.rb | 43 +++++++++++++++++++ 2 files changed, 77 insertions(+) create mode 100644 services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/checker.rb create mode 100644 services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/direction_count/checker.rb diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/checker.rb b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/checker.rb new file mode 100644 index 00000000000..06d7c3b5201 --- /dev/null +++ b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/checker.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module Evidence + module GenAI + module Evaluation + class Checker < ApplicationService + Result = Data.define(:optimal, :failures) + + CHECKS = [ + Evidence::GenAI::Evaluation::DirectionCount::Checker + ] + + attr_reader :feedback + + def initialize(feedback:) + @feedback = feedback + end + + def run + failures = [] + + CHECKS.each do |check| + result = check.run(feedback:) + failures += result.rules unless result.optimal + end + + optimal = failures.empty? + + Result.new(optimal:, failures:) + end + end + end + end +end diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/direction_count/checker.rb b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/direction_count/checker.rb new file mode 100644 index 00000000000..459d05dfa5f --- /dev/null +++ b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/direction_count/checker.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module Evidence + module GenAI + module Evaluation + module DirectionCount + class Checker < ApplicationService + CHAT_API = Evidence::Gemini::Chat + KEY_OPTIMAL = 'optimal' + + Result = Data.define(:optimal, :rules) + + attr_reader :feedback + + def initialize(feedback:) + @feedback = feedback + end + + def run + return suboptimal if feedback.question_count > 1 + + optimal + + # TODO: Add LLM check for Two Directions + # rescue NoMethodError => e + # Evidence.error_notifier.report(e) + # true + end + + private def optimal = Result.new(optimal: true, rules: []) + private def suboptimal = Result.new(optimal: false, rules: ['two_directions']) + + private def system_prompt = Evidence::GenAI::Evaluation::PromptBuilder.run(prompt:) + private def model = CHAT_API::LARGE_MODEL + + private def llm_response + @llm_response ||= CHAT_API.run(system_prompt:, entry: feedback, model:) + end + end + end + end + end +end From 6c62072a901d27ff2e0593cac36034dc41c19f4a Mon Sep 17 00:00:00 2001 From: Daniel Drabik Date: Wed, 15 Jan 2025 09:55:52 -0500 Subject: [PATCH 07/51] Add some initial regex checks. --- .../gen_ai/evaluation/base_checker.rb | 21 +++++ .../evidence/gen_ai/evaluation/checker.rb | 34 ------- .../gen_ai/evaluation/data_fetcher.rb | 2 - .../evaluation/direction_count/checker.rb | 27 +----- .../gen_ai/evaluation/feedback_evaluator.rb | 38 ++++++++ .../gen_ai/evaluation/grammar/checker.rb | 21 +++++ .../gen_ai/evaluation/tone/checker.rb | 21 +++++ .../gen_ai/evaluation/verbose/checker.rb | 23 +++++ .../evidence/app/services/evidence/scalpel.rb | 88 +++++++++++++++++++ .../lib/core_ext/string_extensions.rb | 16 ++-- .../QuillLMS/lib/tasks/gen_ai_evaluation.thor | 6 +- 11 files changed, 226 insertions(+), 71 deletions(-) create mode 100644 services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/base_checker.rb delete mode 100644 services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/checker.rb create mode 100644 services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/feedback_evaluator.rb create mode 100644 services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/grammar/checker.rb create mode 100644 services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/tone/checker.rb create mode 100644 services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/verbose/checker.rb create mode 100644 services/QuillLMS/engines/evidence/app/services/evidence/scalpel.rb diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/base_checker.rb b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/base_checker.rb new file mode 100644 index 00000000000..de791a5b502 --- /dev/null +++ b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/base_checker.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Evidence + module GenAI + module Evaluation + class BaseChecker < ApplicationService + + Result = Data.define(:optimal, :errors) + + attr_reader :feedback, :entry + + def initialize(feedback:, entry:) + @feedback = feedback + @entry = entry + end + + private def optimal = Result.new(optimal: true, errors: []) + end + end + end +end diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/checker.rb b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/checker.rb deleted file mode 100644 index 06d7c3b5201..00000000000 --- a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/checker.rb +++ /dev/null @@ -1,34 +0,0 @@ -# frozen_string_literal: true - -module Evidence - module GenAI - module Evaluation - class Checker < ApplicationService - Result = Data.define(:optimal, :failures) - - CHECKS = [ - Evidence::GenAI::Evaluation::DirectionCount::Checker - ] - - attr_reader :feedback - - def initialize(feedback:) - @feedback = feedback - end - - def run - failures = [] - - CHECKS.each do |check| - result = check.run(feedback:) - failures += result.rules unless result.optimal - end - - optimal = failures.empty? - - Result.new(optimal:, failures:) - end - end - end - end -end diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/data_fetcher.rb b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/data_fetcher.rb index 03e4246cf6b..338efe139b9 100644 --- a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/data_fetcher.rb +++ b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/data_fetcher.rb @@ -9,8 +9,6 @@ class DataFetcher < ApplicationService DEFAULT_FILE = 'whales_suboptimal.csv' CSV_FILE_PATH = "#{Evidence::Engine.root}/app/services/evidence/gen_ai/evaluation/data/%s" - ALL_RULES = Evidence::GenAI::Evaluation::PromptBuilder - attr_reader :rule, :optimal def initialize(rule: nil, optimal: nil) diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/direction_count/checker.rb b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/direction_count/checker.rb index 459d05dfa5f..f924352fd43 100644 --- a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/direction_count/checker.rb +++ b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/direction_count/checker.rb @@ -4,38 +4,15 @@ module Evidence module GenAI module Evaluation module DirectionCount - class Checker < ApplicationService - CHAT_API = Evidence::Gemini::Chat - KEY_OPTIMAL = 'optimal' - - Result = Data.define(:optimal, :rules) - - attr_reader :feedback - - def initialize(feedback:) - @feedback = feedback - end + class Checker < Evaluation::BaseChecker def run return suboptimal if feedback.question_count > 1 optimal - - # TODO: Add LLM check for Two Directions - # rescue NoMethodError => e - # Evidence.error_notifier.report(e) - # true end - private def optimal = Result.new(optimal: true, rules: []) - private def suboptimal = Result.new(optimal: false, rules: ['two_directions']) - - private def system_prompt = Evidence::GenAI::Evaluation::PromptBuilder.run(prompt:) - private def model = CHAT_API::LARGE_MODEL - - private def llm_response - @llm_response ||= CHAT_API.run(system_prompt:, entry: feedback, model:) - end + private def suboptimal = Result.new(optimal: false, errors: ['two_directions']) end end end diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/feedback_evaluator.rb b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/feedback_evaluator.rb new file mode 100644 index 00000000000..9a93dabe795 --- /dev/null +++ b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/feedback_evaluator.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module Evidence + module GenAI + module Evaluation + class FeedbackEvaluator < ApplicationService + Result = Data.define(:optimal, :errors) + + CHECKS = [ + Tone::Checker, + Grammar::Checker, + Verbose::Checker, + DirectionCount::Checker + ] + + attr_reader :feedback, :entry + + def initialize(feedback:, entry:) + @feedback = feedback + @entry = entry + end + + def run + errors = [] + + CHECKS.each do |check| + result = check.run(feedback:, entry:) + errors += result.errors unless result.optimal + end + + optimal = errors.empty? + + Result.new(optimal:, errors:) + end + end + end + end +end diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/grammar/checker.rb b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/grammar/checker.rb new file mode 100644 index 00000000000..183fdfb3198 --- /dev/null +++ b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/grammar/checker.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Evidence + module GenAI + module Evaluation + module Grammar + class Checker < Evaluation::BaseChecker + GRAMMAR_REGEX = /(grammar|spelling|punctuation)/ + + def run + return suboptimal if feedback.match?(GRAMMAR_REGEX) + + optimal + end + + private def suboptimal = Result.new(optimal: false, errors: ['grammar']) + end + end + end + end +end diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/tone/checker.rb b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/tone/checker.rb new file mode 100644 index 00000000000..8c433ef0088 --- /dev/null +++ b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/tone/checker.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Evidence + module GenAI + module Evaluation + module Tone + class Checker < Evaluation::BaseChecker + TONE_REGEX = /(That\'s incorrect|To make your response better|unrelated|off-topic|Can you|You\'re right|You didn\'t|Please)/ + + def run + return suboptimal if feedback.match?(TONE_REGEX) + + optimal + end + + private def suboptimal = Result.new(optimal: false, errors: ['tone']) + end + end + end + end +end diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/verbose/checker.rb b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/verbose/checker.rb new file mode 100644 index 00000000000..bdd19c6a548 --- /dev/null +++ b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/verbose/checker.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Evidence + module GenAI + module Evaluation + module Verbose + class Checker < Evaluation::BaseChecker + MAX_SIZE = 250 + MAX_SENTENCES = 4 + + def run + return suboptimal if feedback.size > MAX_SIZE + return suboptimal if feedback.sentence_count > MAX_SENTENCES + + optimal + end + + private def suboptimal = Result.new(optimal: false, errors: ['verbose']) + end + end + end + end +end diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/scalpel.rb b/services/QuillLMS/engines/evidence/app/services/evidence/scalpel.rb new file mode 100644 index 00000000000..33a0eceed80 --- /dev/null +++ b/services/QuillLMS/engines/evidence/app/services/evidence/scalpel.rb @@ -0,0 +1,88 @@ +# frozen_string_literal: true +# +# File copied from https://github.com/louismullie/scalpel since RubyGems isn't up to date (and it's only one file) +# +# Sentence segmentation based on a set of predefined +# rules that handle a large number of usage cases of +# sentence enders. The idea is to remove all cases of +# .!? being used for other purposes than marking a +# full stop before naively segmenting the text. +class Evidence::Scalpel + + # Current version. + VERSION = '0.2.2' + + # Segment a text using the Scalpel algorithm. + # This will eventually be ported to a gem. + def self.cut(text) + + # Get a copy of the string. + text = text.to_s + # Remove composite abbreviations. + text.gsub!('et al.', '&&&') + # Remove suspension points. + text.gsub!('...', '&;&.') + # Remove floating point numbers. + text.gsub!(/([0-9]+)\.([0-9]+)/) { $1 + '&@&' + $2 } + # Handle floats without leading zero. + text.gsub!(/\s\.([0-9]+)/) { ' &#&' + $1 } + # Remove abbreviations. + text.gsub!(/(?:[A-Za-z]\.){2,}/) { |abbr| abbr.gsub('.', '&-&') } + # Remove initials. + text.gsub!(/(?:[A-Z]\.)/) {|abbr| abbr.gsub('.', '&£&') } + # Remove titles. + text.gsub!(/[A-Z][a-z]{1,2}\./) { |title| title.gsub('.', '&*&') } + # Unstick sentences from each other. + text.gsub!(/([^.?!]\.|\!|\?)([^\s"'])/) { $1 + ' ' + $2 } + # Remove sentence enders next to quotes. + text.gsub!(/'([.?!])\s?"/) { '&^&' + $1 } + text.gsub!(/'([.?!])\s?”/) { '&*&' + $1 } + text.gsub!(/([.?!])\s?”/) { '&=&' + $1 } + text.gsub!(/([.?!])\s?'"/) { '&,&' + $1 } + text.gsub!(/([.?!])\s?'/) { '&%&' + $1 } + text.gsub!(/([.?!])\s?"/) { '&$&' + $1 } + # Remove sentence enders before parens. + text.gsub!(/([.?!])\s?\)/) { '&€&' + $1 } + # Split on any sentence ender. + sentences = text.split(/([.!?])/) + new_sents = [] + # Join the obtaine slices. + sentences.each_slice(2) do |slice| + new_sents << slice.join('') + end + # Repair the damage we've done. + results = [] + new_sents.each do |sentence| + # Skip whitespace zones. + next if sentence.strip == '' + # Repair composite abbreviations. + sentence.gsub!('&&&', 'et al.') + # Repair initials. + sentence.gsub!("&£&", ".") + # Repair abbreviations. + sentence.gsub!('&-&', '.') + # Repair titles. + sentence.gsub!('&*&', '.') + # Repair suspension points. + sentence.gsub!('&;&.', '...') + # Repair floats. + sentence.gsub!(/([0-9]+)&@&([0-9]+)/) { $1 + '.' + $2 } + # Repair quotes with sentence enders + sentence.gsub!(/&=&([.!?])/) { $1 + '”' } + sentence.gsub!(/&,&([.!?])/) { $1 + "'\"" } + sentence.gsub!(/&%&([.!?])/) { $1 + "'" } + sentence.gsub!(/&\^&([.?!])/) { "'" + $1 + '"' } + sentence.gsub!(/&\*&([.?!])/) { "'" + $1 + '”' } + sentence.gsub!(/&\$&([.!?])/) { $1 + '"' } + # Repair parens with sentence enders + sentence.gsub!(/&€&([.!?])/) { $1 + ')' } + # Repair floats without leading zeros. + sentence.gsub!(/&#&([0-9]+)/) { '.' + $1 } + results << sentence.strip + end + + results + + end + +end diff --git a/services/QuillLMS/engines/evidence/lib/core_ext/string_extensions.rb b/services/QuillLMS/engines/evidence/lib/core_ext/string_extensions.rb index 2f43b8b81a1..200e870e823 100644 --- a/services/QuillLMS/engines/evidence/lib/core_ext/string_extensions.rb +++ b/services/QuillLMS/engines/evidence/lib/core_ext/string_extensions.rb @@ -1,7 +1,6 @@ # frozen_string_literal: true class String - SENTENCE_PATTERN = /(? Date: Fri, 17 Jan 2025 10:14:19 -0500 Subject: [PATCH 08/51] wip --- .../gen_ai/evaluation/base_checker.rb | 25 +++++++ .../gen_ai/evaluation/data_fetcher.rb | 3 +- .../direction_count/prompt_builder.rb | 66 +++++++++++++++++++ .../prompts/2025_01_13_initial.md | 12 ++++ .../evaluation/misacknowledgement/checker.rb | 24 +++++++ .../misacknowledgement/prompt_builder.rb | 50 ++++++++++++++ .../prompts/2025_01_15_initial.md | 12 ++++ .../prompts/2025_01_15_rephrase.md | 18 +++++ .../{ => simple_rules}/prompt_builder.rb | 3 - .../prompts/2025_01_08_initial.md | 0 .../prompts/2025_01_10_only_feedback.md | 0 .../QuillLMS/lib/tasks/gen_ai_evaluation.thor | 12 ++-- 12 files changed, 216 insertions(+), 9 deletions(-) create mode 100644 services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/direction_count/prompt_builder.rb create mode 100644 services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/direction_count/prompts/2025_01_13_initial.md create mode 100644 services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/checker.rb create mode 100644 services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompt_builder.rb create mode 100644 services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompts/2025_01_15_initial.md create mode 100644 services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompts/2025_01_15_rephrase.md rename services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/{ => simple_rules}/prompt_builder.rb (93%) rename services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/{ => simple_rules}/prompts/2025_01_08_initial.md (100%) rename services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/{ => simple_rules}/prompts/2025_01_10_only_feedback.md (100%) diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/base_checker.rb b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/base_checker.rb index de791a5b502..bf4bab1fa7a 100644 --- a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/base_checker.rb +++ b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/base_checker.rb @@ -4,6 +4,9 @@ module Evidence module GenAI module Evaluation class BaseChecker < ApplicationService + CLEAR_RESPONSE_REGEX = /clearing your response/ + CHAT_API = Evidence::Gemini::Chat + OPTIMAL_KEY = 'optimal' Result = Data.define(:optimal, :errors) @@ -14,7 +17,29 @@ def initialize(feedback:, entry:) @entry = entry end + def run + raise NotImplementedError + end + private def optimal = Result.new(optimal: true, errors: []) + + private def acknowledgement + feedback.sentences.find {|s| !s.match?(CLEAR_RESPONSE_REGEX)} + end + + # default to true for malformed responses, e.g. nil, missing key + private def optimal_llm_response? + llm_response.fetch(OPTIMAL_KEY, true) + # rescue NoMethodError => e + # Evidence.error_notifier.report(e) + # true + end + + private def model = CHAT_API::SMALL_MODEL + + private def llm_response + @llm_response ||= CHAT_API.run(system_prompt:, entry: feedback, model:) + end end end end diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/data_fetcher.rb b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/data_fetcher.rb index 338efe139b9..908a7faa49b 100644 --- a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/data_fetcher.rb +++ b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/data_fetcher.rb @@ -4,7 +4,7 @@ module Evidence module GenAI module Evaluation class DataFetcher < ApplicationService - Item = Data.define(:feedback, :optimal_broad, :optimal, :rule) + Item = Data.define(:entry, :feedback, :optimal_broad, :optimal, :rule) DEFAULT_FILE = 'whales_suboptimal.csv' CSV_FILE_PATH = "#{Evidence::Engine.root}/app/services/evidence/gen_ai/evaluation/data/%s" @@ -32,6 +32,7 @@ def run private def item_from_row(row) Item.new( + entry: row['entry'], feedback: row['feedback'], optimal_broad: ActiveModel::Type::Boolean.new.cast(row['optimal_broad']), optimal: ActiveModel::Type::Boolean.new.cast(row['optimal']), diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/direction_count/prompt_builder.rb b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/direction_count/prompt_builder.rb new file mode 100644 index 00000000000..bc8667796c8 --- /dev/null +++ b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/direction_count/prompt_builder.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +module Evidence + module GenAI + module Evaluation + module TwoDirections + class PromptBuilder < ApplicationService + include Evidence::MarkdownFormatter + + TEMPLATE_FOLDER = 'app/services/evidence/gen_ai/evaluation/prompts/' + TEMPLATE = '2025_01_10_only_feedback.md' + + RULES = [ + 'Count the number of sentences that end in a questionDo not ask more that one question. Even one question with two parts is not allowed', + "Ignore question sentences and count the number of next steps given by the feedback. That number should only be one." + ] + + SUBOPTIMAL_EXAMPLES = { + two_questions: [ + "That's true. Why do you think the people revolted? What else may be a factor?" + two_directions: [ + "That's true. Add some detail to your answer about ocean pollution. Also, list companies that are pollutors." + ] + } + + OPTIMAL_EXAMPLES = [ + "It's true that some boys were too young to enlist. Add another detail to make your response more specific. What was the age requirement for soldiers?", + "It's true that many boys did not meet the age requirement. Add another detail to make your response more specific. What was the age requirement for soldiers?", + "Try clearing your response and starting again. What requirement kept some boys from volunteering in the British Army during World War I? Check that your response only uses information from the text.", + "Instead of focusing on a reason why some boys wanted to volunteer during WWI, write about what prevented them from being able to enlist. What prevented some boys from volunteering in the British Army?", + "Try clearing your response and starting again. Instead of talking about how TrailGuard computer vision cameras need radio signal or internet service to work, talk about the cameras' technical limitations. What technical limitation can TrailGuard cameras have?", + "That's true that pollution is bad. What causes pollution to be bad?", + "That's true, blue whales are the largest mammals. How large can they grow to be?", + "That's true! Now add more specific information to your response. What is one main cause of the elephant poaching problem?", + "That's true - the TrailGuard computer vision system can occasionally lose connectivity! Now add a specific detail to your response. Where can TrailGuard cameras occasionally lose connection?" + ] + + private def template_variables + { + rules_list:, + examples_table: + } + end + + def run = template % template_variables + + private def examples_table = markdown_table_rows(examples) + private def examples = optimal_rows + suboptimal_rows + private def optimal_rows = OPTIMAL_EXAMPLES.map{|e| [e,true,'']} + private def suboptimal_rows + SUBOPTIMAL_EXAMPLES + .to_a + .map {|rule, array| array.map{|f| [f, false, rule.to_s]}} + .flatten(1) + end + + private def rules_table = markdown_ul((rules) + private def rules = RULES.stringify_keys.to_a + + private def template = File.read(template_file_path) + private def template_file_path = Evidence::Engine.root.join(TEMPLATE_FOLDER, TEMPLATE) + end + end + end + end +end diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/direction_count/prompts/2025_01_13_initial.md b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/direction_count/prompts/2025_01_13_initial.md new file mode 100644 index 00000000000..0c9e57a67f8 --- /dev/null +++ b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/direction_count/prompts/2025_01_13_initial.md @@ -0,0 +1,12 @@ +You are an expert feedback evaluator. You will analyze a piece of text determine if it meets the rules. + +If it passes all of the rules, return {'optimal' : true}. If it fails any of the rules, return {'optimal' : false} + +### Rules +%s + + +### Examples +|feedback|optimal| +|----|----| +%s diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/checker.rb b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/checker.rb new file mode 100644 index 00000000000..944bd6c66d8 --- /dev/null +++ b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/checker.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Evidence + module GenAI + module Evaluation + module Misacknowledgement + class Checker < Evaluation::BaseChecker + def run + optimal_llm_response? ? optimal : suboptimal + end + + private def suboptimal = Result.new(optimal: false, errors: ['misacknowledgement']) + + private def system_prompt = Evidence::GenAI::Evaluation::Misacknowledgement::PromptBuilder.run(entry:) + + private def model = CHAT_API::LARGE_MODEL + private def llm_response + @llm_response ||= CHAT_API.run(system_prompt:, entry: acknowledgement, model:) + end + end + end + end + end +end diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompt_builder.rb b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompt_builder.rb new file mode 100644 index 00000000000..16850bec39d --- /dev/null +++ b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompt_builder.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +module Evidence + module GenAI + module Evaluation + module Misacknowledgement + class PromptBuilder < ApplicationService + include Evidence::MarkdownFormatter + + TEMPLATE_FOLDER = 'app/services/evidence/gen_ai/evaluation/misacknowledgement/prompts/' + TEMPLATE = '2025_01_15_initial.md' + Example = Data.define(:entry, :feedback, :optimal) + + EXAMPLES = [ + Example.new('the ocean is deep', "It's true that the ocean is deep.", true), + Example.new('the war was long and hard', "It's true, the war was long.", true), + Example.new('there were many people involved in the battle', "It's true that the battle involved many people.", true), + Example.new('the war was long and hard', "It's true war was long.", true), + Example.new('the pasta is made with wheat flour and water', "It's true that pasta has many ingredients, but instead write about the cooking method.", true), + Example.new('the pasta is made with wheat flour and water', "Instead of talking about where pasta was invented, write about the cooking method.", false), + Example.new('the boys were scared.', "It's true that the boys were small and frightened.", false), + Example.new('some gibberish.', "It's true pizza is delicious.", false), + Example.new('the pasta is made with wheat flour and water', "Instead of talking about where pasta was invented, write about the cooking method.", false) + ] + + attr_reader :entry + + def initialize(entry:) + @entry = entry + end + + private def template_variables + { + entry:, + examples_table: + } + end + + private def examples_table = markdown_table_rows(examples) + private def examples = EXAMPLES.map(&:deconstruct) + + def run = template % template_variables + + private def template = File.read(template_file_path) + private def template_file_path = Evidence::Engine.root.join(TEMPLATE_FOLDER, TEMPLATE) + end + end + end + end +end diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompts/2025_01_15_initial.md b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompts/2025_01_15_initial.md new file mode 100644 index 00000000000..1561e2b5a19 --- /dev/null +++ b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompts/2025_01_15_initial.md @@ -0,0 +1,12 @@ +Evaluate whether the chatbot's response acknowledges some of the information in the user's input. Paraphrasing, rewording, or reorganizing the information is acceptable. + +Based on your evaluation, return a JSON response: + +- {"optimal": true} if the chatbot's response accurately reflects the user's input. +- {"optimal": false} if the chatbot's response introduces any unsupported information that isn't in the user's input. + +User Input: + +%s + +Chatbot Response (entry): diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompts/2025_01_15_rephrase.md b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompts/2025_01_15_rephrase.md new file mode 100644 index 00000000000..0dc11897bdf --- /dev/null +++ b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompts/2025_01_15_rephrase.md @@ -0,0 +1,18 @@ +You are an expert feedback evaluator. You will analyze a piece of feedback given and determine if it meets the feedback guidelines. Use the student's answer for reference. + +Student's Answer +``` +%s +``` + +Go through these rules one by one and evaluate the feedback. If the feedback does not meet the criteria of a rule, return JSON {"optimal" : false}. Stop evaluating rule if you hit a failing rule. If the feedback passes all of the rules, return {"optimal" : true} + +## Examples +|user_input|chatbot_response|optimal| +|----|----|----| +%s + +### Rules +1. Return JSON {'optimal' : true} + +Feedback: diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/prompt_builder.rb b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/simple_rules/prompt_builder.rb similarity index 93% rename from services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/prompt_builder.rb rename to services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/simple_rules/prompt_builder.rb index bd40040d464..f826e6b3135 100644 --- a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/prompt_builder.rb +++ b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/simple_rules/prompt_builder.rb @@ -10,13 +10,10 @@ class PromptBuilder < ApplicationService TEMPLATE = '2025_01_10_only_feedback.md' RULES = { - # verbose: 'Do not write more than 4 sentences.', grammar: 'Do not give grammar, spelling, or punctuation feedback.', - two_questions: 'Do not ask more that one question. Even one question with two parts is not allowed', repeats_question: 'Do not repeat the question or talk about the conjunction, e.g. "The sentence says x, so do this", "The conjunction \'but\' was used..."', be_concise: "Do not tell the student to 'Be Concise' or something similar.", mixed_message: 'Do not give conflicting advise, e.g. "good work, now clear your response.", "that\'s true, but is also not true"', - two_directions: "Ignore question sentences and count the number of next steps given by the feedback. That number should only be one." } UNUSED_RULES = { diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/prompts/2025_01_08_initial.md b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/simple_rules/prompts/2025_01_08_initial.md similarity index 100% rename from services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/prompts/2025_01_08_initial.md rename to services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/simple_rules/prompts/2025_01_08_initial.md diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/prompts/2025_01_10_only_feedback.md b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/simple_rules/prompts/2025_01_10_only_feedback.md similarity index 100% rename from services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/prompts/2025_01_10_only_feedback.md rename to services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/simple_rules/prompts/2025_01_10_only_feedback.md diff --git a/services/QuillLMS/lib/tasks/gen_ai_evaluation.thor b/services/QuillLMS/lib/tasks/gen_ai_evaluation.thor index a4e70c69b81..981edefeaf7 100644 --- a/services/QuillLMS/lib/tasks/gen_ai_evaluation.thor +++ b/services/QuillLMS/lib/tasks/gen_ai_evaluation.thor @@ -3,7 +3,7 @@ require_relative '../../config/environment' class GenAIEvaluation < Thor - Result = Data.define(:feedback, :optimal, :optimal_output, :optimal_correct, :rule, :rule_output) + Result = Data.define(:entry, :acknowledgement, :feedback, :optimal, :optimal_output, :optimal_correct, :rule, :rule_output) # bundle exec thor gen_a_i_evaluation:run_trial 5 desc 'run_trial 5', 'Run an feedback evalution trial' @@ -19,7 +19,7 @@ class GenAIEvaluation < Thor optimal_data = Evidence::GenAI::Evaluation::DataFetcher.run(optimal: true).first(limit.to_i) suboptimal_data = Evidence::GenAI::Evaluation::DataFetcher.run(optimal: false, rule:).first(limit.to_i) - data = [optimal_data, suboptimal_data].flatten + data = [suboptimal_data, optimal_data].flatten run_test(data, limit) end @@ -27,6 +27,7 @@ class GenAIEvaluation < Thor # put helper methods in this block no_commands do GEN_AI_OUTPUT_FOLDER = ENV.fetch('GEN_AI_OUTPUT_FOLDER', Rails.root.join('/lib/data/')) + CLEAR_RESPONSE_REGEX = /clearing your response/ private def run_test(data, limit) results = [] @@ -35,14 +36,15 @@ class GenAIEvaluation < Thor optimal = item.optimal feedback = item.feedback rule = item.rule - - response = Evidence::GenAI::Evaluation::FeedbackEvaluator.run(feedback:, entry: '') + entry = item.entry + acknowledgement = feedback.sentences.find {|s| !s.match?(CLEAR_RESPONSE_REGEX)} + response = Evidence::GenAI::Evaluation::FeedbackEvaluator.run(feedback:, entry:) optimal_output = response.optimal optimal_correct = optimal_output == optimal rule_output = response.errors.join(',') - results << Result.new(feedback:, optimal:, optimal_output:, optimal_correct:, rule:, rule_output:) + results << Result.new(entry:, acknowledgement:, feedback:, optimal:, optimal_output:, optimal_correct:, rule:, rule_output:) end print_results(results) From a75efd8a0719b3cd8b04450fb301388f1eb5d8e8 Mon Sep 17 00:00:00 2001 From: Daniel Drabik Date: Fri, 17 Jan 2025 16:17:11 -0500 Subject: [PATCH 09/51] Prompt that flags a good number of examples. --- .../gen_ai/evaluation/feedback_evaluator.rb | 1 + .../evaluation/misacknowledgement/checker.rb | 13 +++++++++++- .../misacknowledgement/prompt_builder.rb | 16 ++++++++++---- .../prompts/2025_01_17_explain_feedback.md | 21 +++++++++++++++++++ .../prompts/2025_01_17_info_in_entry.md | 13 ++++++++++++ .../prompts/2025_01_17_iteration.md | 11 ++++++++++ .../prompts/2025_01_17_iteration_no_reason.md | 8 +++++++ .../lib/core_ext/string_extensions.rb | 4 ++++ 8 files changed, 82 insertions(+), 5 deletions(-) create mode 100644 services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompts/2025_01_17_explain_feedback.md create mode 100644 services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompts/2025_01_17_info_in_entry.md create mode 100644 services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompts/2025_01_17_iteration.md create mode 100644 services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompts/2025_01_17_iteration_no_reason.md diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/feedback_evaluator.rb b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/feedback_evaluator.rb index 9a93dabe795..597de1e88d4 100644 --- a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/feedback_evaluator.rb +++ b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/feedback_evaluator.rb @@ -7,6 +7,7 @@ class FeedbackEvaluator < ApplicationService Result = Data.define(:optimal, :errors) CHECKS = [ + Misacknowledgement::Checker, Tone::Checker, Grammar::Checker, Verbose::Checker, diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/checker.rb b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/checker.rb index 944bd6c66d8..18ee1e9a127 100644 --- a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/checker.rb +++ b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/checker.rb @@ -6,16 +6,27 @@ module Evaluation module Misacknowledgement class Checker < Evaluation::BaseChecker def run + puts '---------' + puts llm_entry + puts llm_response + puts '---------' + optimal_llm_response? ? optimal : suboptimal end + def stem = "CETI needs a large amount of data to effectively train its large language models, so" + private def suboptimal = Result.new(optimal: false, errors: ['misacknowledgement']) private def system_prompt = Evidence::GenAI::Evaluation::Misacknowledgement::PromptBuilder.run(entry:) private def model = CHAT_API::LARGE_MODEL + + private def delimiter = PromptBuilder::DELIMITER + + private def llm_entry = [entry, delimiter, acknowledgement].join('') private def llm_response - @llm_response ||= CHAT_API.run(system_prompt:, entry: acknowledgement, model:) + @llm_response ||= CHAT_API.run(system_prompt:, entry: llm_entry, model:) end end end diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompt_builder.rb b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompt_builder.rb index 16850bec39d..1abfa67829f 100644 --- a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompt_builder.rb +++ b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompt_builder.rb @@ -8,13 +8,18 @@ class PromptBuilder < ApplicationService include Evidence::MarkdownFormatter TEMPLATE_FOLDER = 'app/services/evidence/gen_ai/evaluation/misacknowledgement/prompts/' - TEMPLATE = '2025_01_15_initial.md' + TEMPLATE = '2025_01_17_iteration.md' + DELIMITER = ' :: ' + + Example = Data.define(:entry, :feedback, :optimal) EXAMPLES = [ Example.new('the ocean is deep', "It's true that the ocean is deep.", true), Example.new('the war was long and hard', "It's true, the war was long.", true), + Example.new('they went to the ballgame', "It's true that Tom and his friends went to the ballgame.", true), Example.new('there were many people involved in the battle', "It's true that the battle involved many people.", true), + Example.new('they wanted a manned mission to space', "Instead of writing about NASA wanted a manned mission to space, focus on the budget issues in the text.", true), Example.new('the war was long and hard', "It's true war was long.", true), Example.new('the pasta is made with wheat flour and water', "It's true that pasta has many ingredients, but instead write about the cooking method.", true), Example.new('the pasta is made with wheat flour and water', "Instead of talking about where pasta was invented, write about the cooking method.", false), @@ -31,13 +36,16 @@ def initialize(entry:) private def template_variables { - entry:, - examples_table: + # entry:, + examples_table:, + delimiter: } end + private def delimiter = DELIMITER + private def examples_table = markdown_table_rows(examples) - private def examples = EXAMPLES.map(&:deconstruct) + private def examples = EXAMPLES.map{|e| [[e.entry,delimiter,e.feedback].join(''), e.optimal]} def run = template % template_variables diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompts/2025_01_17_explain_feedback.md b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompts/2025_01_17_explain_feedback.md new file mode 100644 index 00000000000..72447b67f9e --- /dev/null +++ b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompts/2025_01_17_explain_feedback.md @@ -0,0 +1,21 @@ +You are a chatbot response evaluator. + +Based on your evaluation, return a JSON response with three keys 'optimal', 'reason', and 'acknowledgement'. + +- {"optimal": true} if the chatbot's response accurately reflects the user's input without introducing unsupported details, assumptions, or altering the meaning. +- {"optimal": false} if the chatbot's response does not reflect the information in the user input. + +The 'reason' key should be text describing why you chose optimal true or false. Please give a 'reason' in all cases, even if 'optimal' is true. + +The entry will be of the form "user_input%schatbot_response". + +Steps +1. Separate the user_input from the chatbot_response +2. Determine the acknowledgement, the subsection of the chatbot_response that acknowledges what the user wrote. +3. Compare the acknowledgement and what the user wrote, and determine if the acknowledgement generally reflects what the user wrote +4. Return the acknowledgement, optimal, and reason keys. + +## Examples +|entry|optimal| +|----|----| +%s diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompts/2025_01_17_info_in_entry.md b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompts/2025_01_17_info_in_entry.md new file mode 100644 index 00000000000..1224fdd362b --- /dev/null +++ b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompts/2025_01_17_info_in_entry.md @@ -0,0 +1,13 @@ +Evaluate whether the chatbot's response introduces any new information, assumptions, or changes in meaning that are not explicitly stated or implied in the user's input. Paraphrasing is acceptable. You can also ignore transitional or conversational words such as "Instead of writing" or "That's true" and focus only on the facts presented. + +Based on your evaluation, return a JSON response with two keys 'optimal' and 'reason'. + +- {"optimal": true} if the chatbot's response accurately reflects the user's input without introducing unsupported details, assumptions, or altering the meaning. +- {"optimal": false} if the chatbot's response does not reflect the information in the user input. + +The 'reason' key should be text describing why you chose optimal true or false. Please give a 'reason' in all cases, even if 'optimal' is true. + +Be strict in identifying inaccuracies, but do not flag stylistic differences or paraphrasing that maintains the original meaning. Return only the JSON output. + +The entry will be of the form "user_input%schatbot_response". + diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompts/2025_01_17_iteration.md b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompts/2025_01_17_iteration.md new file mode 100644 index 00000000000..9df0a4ae2c4 --- /dev/null +++ b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompts/2025_01_17_iteration.md @@ -0,0 +1,11 @@ +You are an expert feedback evaluator. The feedback that you are analyzing should acknowledge what the user wrote, and give further instructions. You are to ignore the feedback's further instructions and just analyze the feedback's acknowledgement of what the user wrote, e.g. ignore the instructions in the feedback such as, "Instead of writing", "That's true", "focus on this", "write about this.". You can also assume that pronouns in the user_input reference proper nouns or nouns in the feedback (or vice versa). + +Based on your evaluation, return a JSON response with two keys 'optimal' and 'reason'. + +- {"optimal": true} if the feedback accurately acknowledges the user input. Paraphrasing and rewording is ok. The acknowledgement does not need to contain all the information in the user input. Default to labeling {"optimal": true} unless you have a clear reason not to. +- {"optimal": false} if the acknowledgement in the feedback does not reflect the information in the user input. + +The 'reason' key should be text describing why you chose optimal true or false. Please give a 'reason' in all cases, even if 'optimal' is true. + +The entry will be of the form "user_input%schatbot_response". + diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompts/2025_01_17_iteration_no_reason.md b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompts/2025_01_17_iteration_no_reason.md new file mode 100644 index 00000000000..0b722906fd9 --- /dev/null +++ b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompts/2025_01_17_iteration_no_reason.md @@ -0,0 +1,8 @@ +You are an expert feedback evaluator. The feedback that you are analyzing should acknowledge what the user wrote, and give further instructions. You are to ignore the feedback's further instructions and just analyze the feedback's acknowledgement of what the user wrote, e.g. ignore the instructions in the feedback such as, "Instead of writing", "That's true", "focus on this", "write about this.". You can also assume that pronouns in the user_input reference proper nouns or nouns in the feedback (or vice versa). + +Based on your evaluation, return a JSON response with one key 'optimal'. + +- {"optimal": true} if the feedback accurately acknowledges the user input. Paraphrasing and rewording is ok. The acknowledgement does not need to contain all the information in the user input. Default to labeling {"optimal": true} unless you have a clear reason not to. +- {"optimal": false} if the acknowledgement in the feedback does not reflect the information in the user input. + +The entry will be of the form "user_input%schatbot_response". diff --git a/services/QuillLMS/engines/evidence/lib/core_ext/string_extensions.rb b/services/QuillLMS/engines/evidence/lib/core_ext/string_extensions.rb index 200e870e823..215b5e385cd 100644 --- a/services/QuillLMS/engines/evidence/lib/core_ext/string_extensions.rb +++ b/services/QuillLMS/engines/evidence/lib/core_ext/string_extensions.rb @@ -23,4 +23,8 @@ def question_count count + questions_in_sentence end end + + def string_before(delimiter) + include?(delimiter) ? split(delimiter, 2).first : self + end end From 071fbd37d9a3a5ee3aff850979546d2596f00a42 Mon Sep 17 00:00:00 2001 From: Daniel Drabik Date: Fri, 17 Jan 2025 16:33:54 -0500 Subject: [PATCH 10/51] More iteration. --- .../evaluation/misacknowledgement/prompt_builder.rb | 3 ++- .../prompts/2025_01_17_iteration.md | 2 +- .../prompts/2025_01_17_iteration_with_examples.md | 13 +++++++++++++ 3 files changed, 16 insertions(+), 2 deletions(-) create mode 100644 services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompts/2025_01_17_iteration_with_examples.md diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompt_builder.rb b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompt_builder.rb index 1abfa67829f..f83376c9e78 100644 --- a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompt_builder.rb +++ b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompt_builder.rb @@ -8,7 +8,7 @@ class PromptBuilder < ApplicationService include Evidence::MarkdownFormatter TEMPLATE_FOLDER = 'app/services/evidence/gen_ai/evaluation/misacknowledgement/prompts/' - TEMPLATE = '2025_01_17_iteration.md' + TEMPLATE = '2025_01_17_iteration_with_examples.md' DELIMITER = ' :: ' @@ -16,6 +16,7 @@ class PromptBuilder < ApplicationService EXAMPLES = [ Example.new('the ocean is deep', "It's true that the ocean is deep.", true), + Example.new('the ocean is deep', "Instead of explaining that the ocean is deep, write about the issue with ocean pollution.", true), Example.new('the war was long and hard', "It's true, the war was long.", true), Example.new('they went to the ballgame', "It's true that Tom and his friends went to the ballgame.", true), Example.new('there were many people involved in the battle', "It's true that the battle involved many people.", true), diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompts/2025_01_17_iteration.md b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompts/2025_01_17_iteration.md index 9df0a4ae2c4..c74612ce2ae 100644 --- a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompts/2025_01_17_iteration.md +++ b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompts/2025_01_17_iteration.md @@ -1,4 +1,4 @@ -You are an expert feedback evaluator. The feedback that you are analyzing should acknowledge what the user wrote, and give further instructions. You are to ignore the feedback's further instructions and just analyze the feedback's acknowledgement of what the user wrote, e.g. ignore the instructions in the feedback such as, "Instead of writing", "That's true", "focus on this", "write about this.". You can also assume that pronouns in the user_input reference proper nouns or nouns in the feedback (or vice versa). +You are an expert feedback evaluator. The feedback that you are analyzing should acknowledge what the user wrote, and give further instructions. You are to ignore the feedback's further instructions and just analyze the feedback's acknowledgement of what the user wrote, e.g. ignore the instructions in the feedback such as, "Instead of writing", "That's true", "focus on this", "write about this.". Based on your evaluation, return a JSON response with two keys 'optimal' and 'reason'. diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompts/2025_01_17_iteration_with_examples.md b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompts/2025_01_17_iteration_with_examples.md new file mode 100644 index 00000000000..4d7c6ce228e --- /dev/null +++ b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompts/2025_01_17_iteration_with_examples.md @@ -0,0 +1,13 @@ +You are an expert feedback evaluator. The feedback that you are analyzing should acknowledge what the user wrote, and give further instructions. You are to ignore the feedback's further instructions and just analyze the feedback's acknowledgement of what the user wrote, e.g. ignore the instructions in the feedback such as, "Instead of writing", "That's true", "focus on this", "write about this.". You can also assume that pronouns in the user_input reference proper nouns or nouns in the feedback (or vice versa). + +Based on your evaluation, return a JSON response with one key 'optimal'. + +- {"optimal": true} if the feedback accurately acknowledges the user input. Paraphrasing and rewording is ok. The acknowledgement does not need to contain all the information in the user input. Default to labeling {"optimal": true} unless you have a clear reason not to. +- {"optimal": false} if the acknowledgement in the feedback does not reflect the information in the user input. + +The entry will be of the form "user_input%schatbot_response". + +## Examples +|entry|optimal| +|----|----| +%s From 96b6148079388c5cdc52feeda40ba16db6716c31 Mon Sep 17 00:00:00 2001 From: Daniel Drabik Date: Fri, 17 Jan 2025 16:35:45 -0500 Subject: [PATCH 11/51] Remove unused code for now. --- .../direction_count/prompt_builder.rb | 66 ------------------- .../prompts/2025_01_13_initial.md | 12 ---- 2 files changed, 78 deletions(-) delete mode 100644 services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/direction_count/prompt_builder.rb delete mode 100644 services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/direction_count/prompts/2025_01_13_initial.md diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/direction_count/prompt_builder.rb b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/direction_count/prompt_builder.rb deleted file mode 100644 index bc8667796c8..00000000000 --- a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/direction_count/prompt_builder.rb +++ /dev/null @@ -1,66 +0,0 @@ -# frozen_string_literal: true - -module Evidence - module GenAI - module Evaluation - module TwoDirections - class PromptBuilder < ApplicationService - include Evidence::MarkdownFormatter - - TEMPLATE_FOLDER = 'app/services/evidence/gen_ai/evaluation/prompts/' - TEMPLATE = '2025_01_10_only_feedback.md' - - RULES = [ - 'Count the number of sentences that end in a questionDo not ask more that one question. Even one question with two parts is not allowed', - "Ignore question sentences and count the number of next steps given by the feedback. That number should only be one." - ] - - SUBOPTIMAL_EXAMPLES = { - two_questions: [ - "That's true. Why do you think the people revolted? What else may be a factor?" - two_directions: [ - "That's true. Add some detail to your answer about ocean pollution. Also, list companies that are pollutors." - ] - } - - OPTIMAL_EXAMPLES = [ - "It's true that some boys were too young to enlist. Add another detail to make your response more specific. What was the age requirement for soldiers?", - "It's true that many boys did not meet the age requirement. Add another detail to make your response more specific. What was the age requirement for soldiers?", - "Try clearing your response and starting again. What requirement kept some boys from volunteering in the British Army during World War I? Check that your response only uses information from the text.", - "Instead of focusing on a reason why some boys wanted to volunteer during WWI, write about what prevented them from being able to enlist. What prevented some boys from volunteering in the British Army?", - "Try clearing your response and starting again. Instead of talking about how TrailGuard computer vision cameras need radio signal or internet service to work, talk about the cameras' technical limitations. What technical limitation can TrailGuard cameras have?", - "That's true that pollution is bad. What causes pollution to be bad?", - "That's true, blue whales are the largest mammals. How large can they grow to be?", - "That's true! Now add more specific information to your response. What is one main cause of the elephant poaching problem?", - "That's true - the TrailGuard computer vision system can occasionally lose connectivity! Now add a specific detail to your response. Where can TrailGuard cameras occasionally lose connection?" - ] - - private def template_variables - { - rules_list:, - examples_table: - } - end - - def run = template % template_variables - - private def examples_table = markdown_table_rows(examples) - private def examples = optimal_rows + suboptimal_rows - private def optimal_rows = OPTIMAL_EXAMPLES.map{|e| [e,true,'']} - private def suboptimal_rows - SUBOPTIMAL_EXAMPLES - .to_a - .map {|rule, array| array.map{|f| [f, false, rule.to_s]}} - .flatten(1) - end - - private def rules_table = markdown_ul((rules) - private def rules = RULES.stringify_keys.to_a - - private def template = File.read(template_file_path) - private def template_file_path = Evidence::Engine.root.join(TEMPLATE_FOLDER, TEMPLATE) - end - end - end - end -end diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/direction_count/prompts/2025_01_13_initial.md b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/direction_count/prompts/2025_01_13_initial.md deleted file mode 100644 index 0c9e57a67f8..00000000000 --- a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/direction_count/prompts/2025_01_13_initial.md +++ /dev/null @@ -1,12 +0,0 @@ -You are an expert feedback evaluator. You will analyze a piece of text determine if it meets the rules. - -If it passes all of the rules, return {'optimal' : true}. If it fails any of the rules, return {'optimal' : false} - -### Rules -%s - - -### Examples -|feedback|optimal| -|----|----| -%s From 48109ad9d3e745aa4d996bfb498250e7231696c6 Mon Sep 17 00:00:00 2001 From: Daniel Drabik Date: Fri, 17 Jan 2025 16:48:52 -0500 Subject: [PATCH 12/51] Extract concern from base class. --- .../concerns/evidence/llm_requestable.rb | 24 +++++++++++++++++++ .../gen_ai/evaluation/base_checker.rb | 17 +------------ .../evaluation/misacknowledgement/checker.rb | 19 ++++----------- .../evidence/config/initializers/zeitwerk.rb | 1 + 4 files changed, 30 insertions(+), 31 deletions(-) create mode 100644 services/QuillLMS/engines/evidence/app/services/concerns/evidence/llm_requestable.rb diff --git a/services/QuillLMS/engines/evidence/app/services/concerns/evidence/llm_requestable.rb b/services/QuillLMS/engines/evidence/app/services/concerns/evidence/llm_requestable.rb new file mode 100644 index 00000000000..21b00866101 --- /dev/null +++ b/services/QuillLMS/engines/evidence/app/services/concerns/evidence/llm_requestable.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Evidence + module LlmRequestable + extend ActiveSupport::Concern + + CHAT_API = Evidence::Gemini::Chat + OPTIMAL_KEY = 'optimal' + + # default to true for malformed responses, e.g. nil, missing key + private def optimal_llm_response? + llm_response.fetch(OPTIMAL_KEY, true) + rescue NoMethodError => e + Evidence.error_notifier.report(e) + true + end + + private def model = CHAT_API::SMALL_MODEL + + private def llm_response + @llm_response ||= CHAT_API.run(system_prompt:, entry: llm_entry, model:) + end + end +end diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/base_checker.rb b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/base_checker.rb index bf4bab1fa7a..644420c152c 100644 --- a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/base_checker.rb +++ b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/base_checker.rb @@ -5,8 +5,7 @@ module GenAI module Evaluation class BaseChecker < ApplicationService CLEAR_RESPONSE_REGEX = /clearing your response/ - CHAT_API = Evidence::Gemini::Chat - OPTIMAL_KEY = 'optimal' + Result = Data.define(:optimal, :errors) @@ -26,20 +25,6 @@ def run private def acknowledgement feedback.sentences.find {|s| !s.match?(CLEAR_RESPONSE_REGEX)} end - - # default to true for malformed responses, e.g. nil, missing key - private def optimal_llm_response? - llm_response.fetch(OPTIMAL_KEY, true) - # rescue NoMethodError => e - # Evidence.error_notifier.report(e) - # true - end - - private def model = CHAT_API::SMALL_MODEL - - private def llm_response - @llm_response ||= CHAT_API.run(system_prompt:, entry: feedback, model:) - end end end end diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/checker.rb b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/checker.rb index 18ee1e9a127..14e8a51d391 100644 --- a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/checker.rb +++ b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/checker.rb @@ -5,29 +5,18 @@ module GenAI module Evaluation module Misacknowledgement class Checker < Evaluation::BaseChecker - def run - puts '---------' - puts llm_entry - puts llm_response - puts '---------' + include Evidence::LlmRequestable + def run optimal_llm_response? ? optimal : suboptimal end - def stem = "CETI needs a large amount of data to effectively train its large language models, so" - private def suboptimal = Result.new(optimal: false, errors: ['misacknowledgement']) - - private def system_prompt = Evidence::GenAI::Evaluation::Misacknowledgement::PromptBuilder.run(entry:) - - private def model = CHAT_API::LARGE_MODEL - private def delimiter = PromptBuilder::DELIMITER private def llm_entry = [entry, delimiter, acknowledgement].join('') - private def llm_response - @llm_response ||= CHAT_API.run(system_prompt:, entry: llm_entry, model:) - end + private def system_prompt = Evidence::GenAI::Evaluation::Misacknowledgement::PromptBuilder.run(entry:) + private def model = CHAT_API::LARGE_MODEL end end end diff --git a/services/QuillLMS/engines/evidence/config/initializers/zeitwerk.rb b/services/QuillLMS/engines/evidence/config/initializers/zeitwerk.rb index 18cab5bb936..ba911d79bca 100644 --- a/services/QuillLMS/engines/evidence/config/initializers/zeitwerk.rb +++ b/services/QuillLMS/engines/evidence/config/initializers/zeitwerk.rb @@ -25,6 +25,7 @@ 'llm_prompt_relevant_text' => 'LLMPromptRelevantText', 'llm_prompt_template' => 'LLMPromptTemplate', 'llm_prompt_templates_controller' => 'LLMPromptTemplatesController', + 'llm_requestable' => 'LLMRequestable', 'llms' => 'LLMs', 'llms_controller' => 'LLMsController', 'malformed_json_fixer' => 'MalformedJSONFixer', From 611e922d0ea4333875787e91ca1db3112c1a48fc Mon Sep 17 00:00:00 2001 From: Daniel Drabik Date: Fri, 17 Jan 2025 16:50:39 -0500 Subject: [PATCH 13/51] Move more out of the base class. --- .../app/services/concerns/evidence/llm_requestable.rb | 1 + .../services/evidence/gen_ai/evaluation/base_checker.rb | 7 ------- .../gen_ai/evaluation/misacknowledgement/checker.rb | 6 ++++++ .../gen_ai/evaluation/misacknowledgement/prompt_builder.rb | 1 - 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/services/QuillLMS/engines/evidence/app/services/concerns/evidence/llm_requestable.rb b/services/QuillLMS/engines/evidence/app/services/concerns/evidence/llm_requestable.rb index 21b00866101..4b22a9aa053 100644 --- a/services/QuillLMS/engines/evidence/app/services/concerns/evidence/llm_requestable.rb +++ b/services/QuillLMS/engines/evidence/app/services/concerns/evidence/llm_requestable.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true module Evidence + # TODO: Get inflection to work properly. module LlmRequestable extend ActiveSupport::Concern diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/base_checker.rb b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/base_checker.rb index 644420c152c..cd4261f61e1 100644 --- a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/base_checker.rb +++ b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/base_checker.rb @@ -4,9 +4,6 @@ module Evidence module GenAI module Evaluation class BaseChecker < ApplicationService - CLEAR_RESPONSE_REGEX = /clearing your response/ - - Result = Data.define(:optimal, :errors) attr_reader :feedback, :entry @@ -21,10 +18,6 @@ def run end private def optimal = Result.new(optimal: true, errors: []) - - private def acknowledgement - feedback.sentences.find {|s| !s.match?(CLEAR_RESPONSE_REGEX)} - end end end end diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/checker.rb b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/checker.rb index 14e8a51d391..db7f8107c6c 100644 --- a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/checker.rb +++ b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/checker.rb @@ -7,6 +7,8 @@ module Misacknowledgement class Checker < Evaluation::BaseChecker include Evidence::LlmRequestable + CLEAR_RESPONSE_REGEX = /clearing your response/ + def run optimal_llm_response? ? optimal : suboptimal end @@ -17,6 +19,10 @@ def run private def llm_entry = [entry, delimiter, acknowledgement].join('') private def system_prompt = Evidence::GenAI::Evaluation::Misacknowledgement::PromptBuilder.run(entry:) private def model = CHAT_API::LARGE_MODEL + + private def acknowledgement + feedback.sentences.find {|s| !s.match?(CLEAR_RESPONSE_REGEX)} + end end end end diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompt_builder.rb b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompt_builder.rb index f83376c9e78..f404a948159 100644 --- a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompt_builder.rb +++ b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompt_builder.rb @@ -37,7 +37,6 @@ def initialize(entry:) private def template_variables { - # entry:, examples_table:, delimiter: } From 724178c13fb6340c3b0fdefcb6e4b26378dba725 Mon Sep 17 00:00:00 2001 From: Daniel Drabik Date: Sun, 19 Jan 2025 15:08:45 -0500 Subject: [PATCH 14/51] Clean up unused files. --- .../evaluation/simple_rules/prompt_builder.rb | 86 ------------------- .../prompts/2025_01_08_initial.md | 29 ------- .../prompts/2025_01_10_only_feedback.md | 14 --- 3 files changed, 129 deletions(-) delete mode 100644 services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/simple_rules/prompt_builder.rb delete mode 100644 services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/simple_rules/prompts/2025_01_08_initial.md delete mode 100644 services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/simple_rules/prompts/2025_01_10_only_feedback.md diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/simple_rules/prompt_builder.rb b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/simple_rules/prompt_builder.rb deleted file mode 100644 index f826e6b3135..00000000000 --- a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/simple_rules/prompt_builder.rb +++ /dev/null @@ -1,86 +0,0 @@ -# frozen_string_literal: true - -module Evidence - module GenAI - module Evaluation - class PromptBuilder < ApplicationService - include Evidence::MarkdownFormatter - - TEMPLATE_FOLDER = 'app/services/evidence/gen_ai/evaluation/prompts/' - TEMPLATE = '2025_01_10_only_feedback.md' - - RULES = { - grammar: 'Do not give grammar, spelling, or punctuation feedback.', - repeats_question: 'Do not repeat the question or talk about the conjunction, e.g. "The sentence says x, so do this", "The conjunction \'but\' was used..."', - be_concise: "Do not tell the student to 'Be Concise' or something similar.", - mixed_message: 'Do not give conflicting advise, e.g. "good work, now clear your response.", "that\'s true, but is also not true"', - } - - UNUSED_RULES = { - gives_answer: "Only discuss facts in the student's answer, do not give away specific facts or details for the student to use except for acknowledging the text in their answer. Instead, You should redirect them to the text to find a detail or ask them a question about the text", - misacknowledgement: 'If the feedback "Yes, and" the student\'s answer, it must only use the details in the student\'s answer.', - } - - SUBOPTIMAL_EXAMPLES = { - grammar: [ - "That's true. Now correct the spelling of the word 'Their'", - "Nice work. Your answer needs an apostrophe to show possesion, please add." - ], - two_questions: [ - "That's true. Why do you think the people revolted? What else may be a factor?" - ], - repeats_question: [ - "That's true, many underage boys did not last long in the war. However, the sentence starts with 'but,' which indicates a contrasting idea. The question asks what prevented many young teenage boys from volunteering. What was the age requirement for the British Army?" - ], - be_concise: [ - "That's good, now make your answer a little shorter and less wordy." - ], - mixed_message: [ - "Nice work, you are on the right track. Clear your response and start again with details from the passage" - ], - two_directions: [ - "That's true. Add some detail to your answer about ocean pollution. Also, list companies that are pollutors." - ] - } - - OPTIMAL_EXAMPLES = [ - "It's true that some boys were too young to enlist. Add another detail to make your response more specific. What was the age requirement for soldiers?", - "It's true that many boys did not meet the age requirement. Add another detail to make your response more specific. What was the age requirement for soldiers?", - "Try clearing your response and starting again. What requirement kept some boys from volunteering in the British Army during World War I? Check that your response only uses information from the text.", - "Instead of focusing on a reason why some boys wanted to volunteer during WWI, write about what prevented them from being able to enlist. What prevented some boys from volunteering in the British Army?", - "Try clearing your response and starting again. Instead of talking about how TrailGuard computer vision cameras need radio signal or internet service to work, talk about the cameras' technical limitations. What technical limitation can TrailGuard cameras have?", - "That's true that pollution is bad. What causes pollution to be bad?", - "That's true, blue whales are the largest mammals. How large can they grow to be?", - "That's true! Now add more specific information to your response. What is one main cause of the elephant poaching problem?", - "That's true - the TrailGuard computer vision system can occasionally lose connectivity! Now add a specific detail to your response. Where can TrailGuard cameras occasionally lose connection?" - ] - - private def template_variables - { - rules_table:, - examples_table: - } - end - - def run = template % template_variables - - - private def examples_table = markdown_table_rows(examples) - private def examples = optimal_rows + suboptimal_rows - private def optimal_rows = OPTIMAL_EXAMPLES.map{|e| [e,true,'']} - private def suboptimal_rows - SUBOPTIMAL_EXAMPLES - .to_a - .map {|rule, array| array.map{|f| [f, false, rule.to_s]}} - .flatten(1) - end - - private def rules_table = markdown_table_rows(rules) - private def rules = RULES.stringify_keys.to_a - - private def template = File.read(template_file_path) - private def template_file_path = Evidence::Engine.root.join(TEMPLATE_FOLDER, TEMPLATE) - end - end - end -end diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/simple_rules/prompts/2025_01_08_initial.md b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/simple_rules/prompts/2025_01_08_initial.md deleted file mode 100644 index 30b94b02ea3..00000000000 --- a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/simple_rules/prompts/2025_01_08_initial.md +++ /dev/null @@ -1,29 +0,0 @@ -You are an expert feedback evaluator. You will analyze a student's answer and a piece of feedback given and determine if it meets the feedback guidelines. - -Answer -``` -%s -``` - -Go through these rules one by one and evaluate the feedback. If the feedback does not meet the criteria of a rule, return JSON {"optimal" : false, "rule" : "rule_name"} where the "rule_name" is the name of the rule. Stop evaluating rule if you hit a failing rule. If the feedback passes all of the rules, return {"optimal" : true} - -### Rules -|rule|criteria| -|----|----| -%s - - -### Examples -|answer|feedback|optimal|rule| -|----|----|----|----| -|I don't understand the question| Try clearing your response and starting again. What requirement kept some boys from volunteering in the British Army during World War I? Check that your response only uses information from the text.|true|| -|some boys really wanted to enlist| Instead of focusing on a reason why some boys wanted to volunteer during WWI, write about what prevented them from being able to enlist. What prevented some boys from volunteering in the British Army?|true|| -|it's over their|That's true. Now correct the spelling of the word 'Their'|false|grammar| -|ocean pollution is bad|That's true. Add some detail to your answer about ocean pollution. Also, list companies that are pollutors|false|two_directions| -|ocean pollution is bad|That's true that pollution is bad. Why is pollution bad?|true|| -|the pizza is hot|Nice work, you are on the right track. Clear your response and start again with details from the passage|false|mixed_message| -|because blue whales are the largest mammals.|That's true, blue whales are the largest mammals. How large can they grow to?|true|| -|because blue whales eat plankton.|You're on the right track, blue whales are the largest mammals. How large can they grow to?|false|misacknowledgement| -|pizza is delicous.|That's true, pizza is made with cheese. What other ingredients are used?|false|misacknowledgement| -|because blue whales are the largest mammals|You're on the right track. The text also states that Blue whales can weigh 100 tons. Include that in your response|false|gives_answer| -|I tried and tried and tried and tried.|That's good, now make your answer a little shorter and less wordy|false|be_concise| diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/simple_rules/prompts/2025_01_10_only_feedback.md b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/simple_rules/prompts/2025_01_10_only_feedback.md deleted file mode 100644 index 186e462df65..00000000000 --- a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/simple_rules/prompts/2025_01_10_only_feedback.md +++ /dev/null @@ -1,14 +0,0 @@ -You are an expert feedback evaluator. You will analyze a piece of feedback given and determine if it meets the rules. - -Go through these rules one by one and evaluate the feedback. If the feedback does not meet the criteria of a rule, return JSON {"optimal" : false, "rule" : "rule_name"} where the "rule_name" is the name of the rule. Stop evaluating rule if you hit a failing rule. If the feedback passes all of the rules, return {"optimal" : true} - -### Rules -|rule|criteria| -|----|----| -%s - - -### Examples -|feedback|optimal|rule| -|----|----|----| -%s From c11e6fc0a00e6ffaca6f2e3f90bd6daf98c08051 Mon Sep 17 00:00:00 2001 From: Daniel Drabik Date: Wed, 22 Jan 2025 12:02:55 -0500 Subject: [PATCH 15/51] Add new data files. --- .../evaluation/data/underage_soldiers_but.csv | 192 +++++++++ .../{whales_suboptimal.csv => whales_so.csv} | 376 +++++++++--------- .../gen_ai/evaluation/data_fetcher.rb | 8 +- 3 files changed, 385 insertions(+), 191 deletions(-) create mode 100644 services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/data/underage_soldiers_but.csv rename services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/data/{whales_suboptimal.csv => whales_so.csv} (83%) diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/data/underage_soldiers_but.csv b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/data/underage_soldiers_but.csv new file mode 100644 index 00000000000..505be8f296b --- /dev/null +++ b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/data/underage_soldiers_but.csv @@ -0,0 +1,192 @@ +entry,feedback,optimal,rule +age restrictions made is difficult for some.,It's true that age restrictions made it difficult for some boys to enlist. Add another detail to make your response more specific. What was the age requirement for soldiers?,TRUE, +laws existed that prevented them from joining legally.,It's true that laws existed that prevented them from joining legally. Add another detail to make your response more specific. What was the age requirement for soldiers?,TRUE, +Britain has a legal age requirement in place that had to be accounted for.,It's true that Britain had a legal age requirement. Add another detail to make your response more specific. What was the age requirement for soldiers?,TRUE, +some were too young to enlist.,It's true that some boys were too young to enlist. Add another detail to make your response more specific. What was the age requirement for soldiers?,TRUE, +many did not meet the age requirement.,It's true that many boys did not meet the age requirement. Add another detail to make your response more specific. What was the age requirement for soldiers?,TRUE, +some were underage.,It's true that some boys were underage. Add another detail to make your response more specific. What was the age requirement for soldiers?,TRUE, +some did not meet the age requirement and had to use forged documents.,"It's true that some boys had to use forged documents to join the army. Since this sentence uses 'but', write about a contrasting idea about them wanting to enlist instead. What was the age requirement for soldiers?",FALSE,tone +the British army had legal age requirements.,It's true that the British Army had a legal age requirement. Add another detail to make your response more specific. What was the age requirement for soldiers?,TRUE, +many of them didn't look old enough to give fake names and ages.,"It's true that some underage boys may not have looked old enough to give fake names and ages. However, the sentence starts with 'but,' which indicates a contrasting idea. The text mentions that many underage boys *were* able to successfully enlist using fake names and ages. Try focusing on what prevented some boys from enlisting, despite wanting to.",FALSE,verbose +the boys too young to join.,It's true that some boys were too young to enlist. Add another detail to make your response more specific. What was the age requirement for soldiers?,TRUE, +they did not meet the age requirement.,It's true that they did not meet the age requirement. Add another detail to make your response more specific. What was the age requirement for soldiers?,TRUE, +there were laws regarding underage people from enlisting.,It's true that there were laws regarding underage people enlisting. Add another detail to make your response more specific. What was the age requirement for soldiers?,TRUE, +the British Army's legal requirement and societal norm was an obstacle.,It's true that there were obstacles for young boys who wanted to enlist in the British Army during WWI. Add another detail to make your response more specific. What was the age requirement for soldiers? How did parents contribute to the situation?,FALSE,two_directions +they were too young to enter the war.,It's true that some boys were too young to enlist. Add another detail to make your response more specific. What was the age requirement for soldiers?,TRUE, +most could not due to being underage.,It's true that many boys were underage. Add another detail to make your response more specific. What was the age requirement for soldiers?,TRUE, +many of the teenage boys were under age.,It's true that many boys were underage. Add another detail to make your response more specific. What was the age requirement for soldiers?,TRUE, +they were too young or didn’t meet the requirements to fight.,It's true that some boys were too young to enlist. Add another detail to make your response more specific. What was the age requirement for soldiers?,TRUE, +there was a legal age requirement.,It's true that there was a legal age requirement. Add another detail to make your response more specific. What was the age requirement for soldiers?,TRUE, +they had to find a way around the British's legal joining age requirement.,"It's true that underage boys had to find ways around the age requirement. Add another detail to make your response more specific. What was the age requirement, and what happened to those underage boys who were discovered?",FALSE,two_directions +many of were too young.,It's true that many boys were too young to enlist. Add another detail to make your response more specific. What was the age requirement for soldiers?,TRUE, +they were too young for the army's age requirement.,It's true that some boys were too young to enlist. Add another detail to make your response more specific. What was the age requirement for soldiers?,TRUE, +they were not suited for frontline service and did not last long.,That's true---some boys were found to be too small or scared to fight after volunteering for the war. Now be more specific. How many boys were sent home during their their first month of enlistment?,FALSE,misacknowledgement +they did not last long in their conflict.,Some boys did not last long in the British Army during WWI---that's true! Add another detail to your response more specific. How many boys were discharged shortly after their enlistment?,TRUE, +they didn't last very long.,Some boys didn't last too long in the British Army during WWI---that's true! Add another detail to your response more specific. How many boys were discharged shortly after their enlistment?,TRUE, +the two didn't last long in their fight.,Some boys didn't last too long in the British Army during WWI---that's true! Add another detail to your response more specific. How many boys were discharged shortly after their enlistment?,TRUE, +they did not last long in terms of combat.,Some boys didn't last long in the British Army during WWI---that's true! Add another detail to your response more specific. How many boys were discharged shortly after their enlistment?,TRUE, +they did not last long with regards to fighting.,Some boys did not last long in the British Army during WWI---that's true! Add another detail to your response more specific. How many boys were discharged shortly after their enlistment?,TRUE, +they were underage and did not have a long military career.,That's true; many underage boys did not have long military careers. Add another detail from the text to make your response more specific. How many underage soldiers were released from service within their first month?,TRUE, +they were not suited for frontline duty and were soon replaced.,That's true---some boys were found to be too small or scared to fight after volunteering for the war. Now be more specific. How many boys were sent home during their their first month of enlistment?,FALSE,misacknowledgement +they were underage and were not able to last long on the front lines of war.,"That's true; many underage boys did not last long in the war. However, the sentence starts with 'but,' which indicates a contrasting idea. The question asks what prevented many young teenage boys from volunteering. What was the age requirement for the British Army?",FALSE,verbose +they did not last long.,Some boys did not last long in the British Army during WWI---that's true! Add another detail to your response more specific. How many boys were discharged shortly after their enlistment?,TRUE, +they were not old enough to fight and did not last long on the front lines.,That's true---some boys were too young to fight and did not last long in the British Army during WWI. Now be more specific. What was the age requirement for soldiers? How many boys were sent home during their first month of enlistment?,FALSE,two_directions +they were underage and did not last long in the front lines of war.,"That's true; many underage boys did not last long in the British Army during WWI. To make your response better, add a detail from the text about why they were sent home. How many underage soldiers were sent home within their first month of service?",FALSE,two_directions +they were too young to fight in the war and didn't last very long.,It's true that some boys were too young to enlist and that many did not last long in the British Army during WWI. Add another detail to make your response more specific. What was the age requirement for soldiers? How many underage soldiers were discharged within their first month of service?,FALSE,two_directions +they lasted for a short while.,Some boys didn't last long in the British Army during WWI---that's true! Add another detail to your response more specific. How many boys were discharged shortly after their enlistment?,TRUE, +many of them didn't last long in the front lines.,Some boys didn't last long in the British Army during WWI---that's true! Add another detail to your response more specific. How many boys were discharged shortly after their enlistment?,TRUE, +they didn't do very well in combat and didn't last long.,Some boys didn't last too long in the British Army during WWI---that's true! Add another detail to your response more specific. How many boys were discharged shortly after their enlistment?,TRUE, +most of the young adults didn't last long during the war.,Some boys didn't last too long in the British Army during WWI---that's true! Add another detail to your response more specific. How many boys were discharged shortly after their enlistment?,TRUE, +lots of them didn't last in the army for very long,Some boys didn't last long in the British Army during WWI---that's true! Add another detail to your response more specific. How many boys were discharged shortly after their enlistment?,TRUE, +it was not a societal norm for them to do so.,It's true that joining the army at a young age was not the societal norm. Add another detail to make your response more specific. What was the age requirement for soldiers?,FALSE,unhelpful +most didn't last long and they were released from service.,Some boys didn't last long in the British Army during WWI---that's true! Add another detail to your response more specific. How many boys were discharged shortly after their enlistment?,TRUE, +this was quite unusual for boys of their age to participate in war.,It's true that it was unusual for boys to enlist in the British Army during that time---that's helpful historical context. Write about a barrier (or something that made it difficult) some boys faced when wanting to volunteer for the war instead. What usually prevented young boys from joining the army?,TRUE, +many of them didn't last long on the front lines.,Some boys didn't last too long in the British Army during WWI---that's true! Add another detail to your response more specific. How many boys were discharged shortly after their enlistment?,TRUE, +that was not a societal norm.,It's true that enlisting in the British Army at a young age was not the norm at that time---that's helpful historical context. Write about a barrier (or something that made it difficult) some boys faced when wanting to volunteer for the war instead. What usually prevented young boys from joining the army?,TRUE, +it was not usual for them to do so.,It's true that it was unusual for boys to enlist in the British Army during that time---that's helpful historical context. Write about a barrier (or something that made it difficult) some boys faced when wanting to volunteer for the war instead. What usually prevented young boys from joining the army?,TRUE, +it was not typically for boys that young to join the army.,It's true that joining the army at a young age was not the norm at that time---that's helpful historical context. Write about a barrier (or something that made it difficult) some boys faced when wanting to volunteer for the war instead. What usually prevented young boys from joining the army?,TRUE, +it was not customary for them to do so.,It's true that joining the army at a young age was not the norm at that time---that's helpful historical context. Write about a barrier (or something that made it difficult) some boys faced when wanting to volunteer for the war instead. What usually prevented young boys from joining the army?,TRUE, +many of these boys were not able to participate in the war because it was unusual for their age.,It's true that it was unusual for boys to enlist in the British Army during that time---that's helpful historical context. Write about a barrier (or something that made it difficult) some boys faced when wanting to volunteer for the war instead. What usually prevented young boys from joining the army?,TRUE, +being enlisted in the Army at a young age is not normal in society.,It's true that enlisting in the British Army at a young age was not the norm at that time---that's helpful historical context. Write about a barrier (or something that made it difficult) some boys faced when wanting to volunteer for the war instead. What usually prevented young boys from joining the army?,TRUE, +it was not a common thing for them to do so.,It's true that joining the army at a young age was not the norm at that time---that's helpful historical context. Write about a barrier (or something that made it difficult) some boys faced when wanting to volunteer for the war instead. What usually prevented young boys from joining the army?,TRUE, +a good number of these boys were not able to because it was very unusual for their age to participate in war.,It's true that it was unusual for boys to enlist in the British Army during that time---that's helpful historical context. Write about a barrier (or something that made it difficult) some boys faced when wanting to volunteer for the war instead. What usually prevented young boys from joining the army?,TRUE, +"at such a young age, being in the Army was far from the norm in society.",It's true that enlisting in the British Army at a young age was not the norm at that time---that's helpful historical context. Write about a barrier (or something that made it difficult) some boys faced when wanting to volunteer for the war instead. What usually prevented young boys from joining the army?,TRUE, +this is not typical for boys of such a young age to enlist in the military.,It's true that it was unusual for boys to enlist in the British Army during that time---that's helpful historical context. Write about a barrier (or something that made it difficult) some boys faced when wanting to volunteer for the war instead. What usually prevented young boys from joining the army?,TRUE, +joining the army at that age was not normal.,It's true that enlisting in the British Army at a young age was not the norm at that time---that's helpful historical context. Write about a barrier (or something that made it difficult) some boys faced when wanting to volunteer for the war instead. What usually prevented young boys from joining the army?,TRUE, +serving in the military at a young age is not common in society.,It's true that enlisting in the British Army at a young age was not the norm at that time---that's helpful historical context. Write about a barrier (or something that made it difficult) some boys faced when wanting to volunteer for the war instead. What usually prevented young boys from joining the army?,TRUE, +joining it at a young age was not a societal norm becuase they needed to work.,"It's true that joining the army at a young age was not the societal norm. However, the sentence starts with ""but"", which indicates that a contrasting idea should follow. The text mentions an age requirement for joining the army. Can you use that information to complete the sentence?",FALSE,tone +it wasn't a common practice for them to do that.,It's true that joining the army at a young age was not the norm at that time---that's helpful historical context. Write about a barrier (or something that made it difficult) some boys faced when wanting to volunteer for the war instead. What usually prevented young boys from joining the army?,TRUE, +joining the army at a young age wasn't a societal norm.,It's true that enlisting in the British Army at a young age was not the norm at that time---that's helpful historical context. Write about a barrier (or something that made it difficult) some boys faced when wanting to volunteer for the war instead. What usually prevented young boys from joining the army?,TRUE, +enlisting to join the army and fighting in the war was not considered a societal norm.,It's true that enlisting in the British Army at a young age was not the norm at that time---that's helpful historical context. Write about a barrier (or something that made it difficult) some boys faced when wanting to volunteer for the war instead. What usually prevented young boys from joining the army?,TRUE, +"joining the army at a young age was not a societal norm, and so boys were faced with pressure to enlist in their everyday lives as well.","It's true that joining the army at a young age was not a societal norm, and that boys faced pressure to enlist. However, the sentence starts with ""but,"" which indicates a contrasting idea. The prompt asks what prevented many young teenage boys from volunteering. Try focusing on the age requirement and what happened to underage soldiers who were discovered.",FALSE,two_directions +the societal norm goes against joining the army at such a young age.,It's true that joining the army at a young age was not the societal norm. Add another detail to make your response more specific. What was the age requirement for soldiers?,FALSE,unhelpful +it was not acceptable for such young boys to join the army.,It's true that it was not acceptable for young boys to join the army. Add another detail to make your response more specific. What was the age requirement for soldiers?,FALSE,unhelpful +many were discharged after their ages were verified,It's true that some boys were sent home after their ages were verified. Add another specific detail from the text to your response. How did the British Army learn that the boys were too young?,TRUE, +many were sent home because of their age,It's true that many underage boys were sent home. Add another detail from the text to make your response more specific. How did the British Army learn that the boys were underage?,TRUE, +many were sent home after their age confirming was verified,It's true that some boys were sent home after their ages were verified. Add another specific detail from the text to your response. How did the British Army learn that the boys were too young?,TRUE, +many were sent home after their IDs were verified,It's true that some boys were sent home after their ages were verified. Add another specific detail from the text to your response. How did the British Army learn that the boys were too young?,TRUE, +many were sent home due to age verification,It's true that some boys were sent home after their ages were verified. Add another specific detail from the text to your response. How did the British Army learn that the boys were too young?,TRUE, +many were sent home after their ages were found out,It's true that some boys were sent home after their ages were verified. Add another specific detail from the text to your response. How did the British Army learn that the boys were too young?,TRUE, +many were sent home after their ages confirming was verified,It's true that some boys were sent home after their ages were verified. Add another specific detail from the text to your response. How did the British Army learn that the boys were too young?,TRUE, +many were sent home due to not being of age,It's true that many underage boys were sent home. Add another detail from the text to make your response more specific. How did the army find out that these boys were underage?,TRUE, +it was not quite normal for such young boys to join the army.,It's true that it was not the norm for young boys to enlist in the British Army during that time---that's helpful historical context. Write about a barrier (or something that made it difficult) some boys faced when wanting to volunteer for the war instead. What usually prevented young boys from joining the army?,TRUE, +many of them were sent home after their ages were verified,It's true that some boys were sent home after their ages were verified. Add another specific detail from the text to your response. How did the British Army learn that the boys were too young?,TRUE, +many were returned home after their ages were verified,It's true that some boys were sent home after their ages were verified. Add another specific detail from the text to your response. How did the British Army learn that the boys were too young?,TRUE, +many were withdrawn after their ages were found out,It's true that some boys were sent home after their ages were verified. Add another specific detail from the text to your response. How did the British Army learn that the boys were too young?,TRUE, +many young people were sent home after their ages were verified.,It's true that some boys were sent home after their ages were verified. Add another specific detail from the text to your response. How did the British Army learn that the boys were too young?,TRUE, +a large number of people were sent home after their true ages were discovered.,It's true that some boys were sent home after their ages were verified. Add another specific detail from the text to your response. How did the British Army learn that the boys were too young?,TRUE, +most of them didn't make it due to their size and actual age.,That's true---some boys were found to be too small or scared to fight after volunteering for the war. Now be more specific. How many boys were sent home during their their first month of enlistment?,TRUE, +they were too young/weak to fight or too scared.,That's true---some boys were found to be too small or scared to fight after volunteering for the war. Now be more specific. How many boys were sent home during their their first month of enlistment?,TRUE, +joining the army at a young age was not considered normal.,It's true that enlisting in the British Army at a young age was not the norm at that time---that's helpful historical context. Write about a barrier (or something that made it difficult) some boys faced when wanting to volunteer for the war instead. What usually prevented young boys from joining the army?,TRUE, +many of them didn't last long because they were too small to fight or too scared.,That's true---some boys were found to be too small or scared to fight after volunteering for the war. Now be more specific. How many boys were sent home during their their first month of enlistment?,TRUE, +many children were sent home after their ages were discovered.,It's true that some boys were sent home after their ages were verified. Add another specific detail from the text to your response. How did the British Army learn that the boys were too young?,TRUE, +many were kicked out after their identity were discovered,It's true that some boys were sent home after their true age was discovered. Add another specific detail from the text to your response. How did the British Army learn that the boys were too young?,TRUE, +many of them would get sent back home because of their age.,It's true that many underage boys were sent home. Add another detail from the text to make your response more specific. How did the army find out their real ages?,TRUE, +many of them were sent home after their ages were found out.,It's true that some boys were sent home after their ages were verified. Add another specific detail from the text to your response. How did the British Army learn that the boys were too young?,TRUE, +many were kicked out after their identities were discovered,It's true that some boys were sent home after their true ages were discovered. Add another specific detail from the text to your response. How did the British Army learn that the boys were too young?,TRUE, +they were too small to fight and too scared.,That's true---some boys were found to be too small or scared to fight after volunteering for the war. Now be more specific. How many boys were sent home during their their first month of enlistment?,TRUE, +many were too small physically.,That's true---some boys were found to be too small or scared to fight after volunteering for the war. Now be more specific. How many boys were sent home during their their first month of enlistment?,TRUE, +many were fired after their agess were found out,It's true that some boys were sent home after their ages were verified. Add another specific detail from the text to your response. How did the British Army learn that the boys were too young?,TRUE, +they were not big enough to fight or too scared.,That's true---some boys were found to be too small or scared to fight after volunteering for the war. Now be more specific. How many boys were sent home during their their first month of enlistment?,TRUE, +many were too small in terms of their physical stature.,That's true---some boys were found to be too small or scared to fight after volunteering for the war. Now be more specific. How many boys were sent home during their their first month of enlistment?,TRUE, +many were too small in stature.,That's true---some boys were found to be too small or scared to fight after volunteering for the war. Now be more specific. How many boys were sent home during their their first month of enlistment?,TRUE, +many were too small in size.,That's true---some boys were found to be too small or scared to fight after volunteering for the war. Now be more specific. How many boys were sent home during their their first month of enlistment?,TRUE, +a great many of them were physically small.,That's true---some boys were found to be too small or scared to fight after volunteering for the war. Now be more specific. How many boys were sent home during their their first month of enlistment?,TRUE, +they were not big enough to fight or to be scared.,That's true---some boys were found to be too small or scared to fight after volunteering for the war. Now be more specific. How many boys were sent home during their their first month of enlistment?,TRUE, +they were not able to enlist,It's true that they were not able to enlist. Add another detail from the text to make your response more specific. What prevented some boys from volunteering in the British Army?,TRUE, +they weren't able to get in,It's true that they weren't able to enlist. Add another detail from the text to make your response more specific. What prevented some boys from volunteering in the British Army?,TRUE, +they were not strong or courageous enough.,That's true---some boys were found to be too small or scared to fight after volunteering for the war. Now be more specific. How many boys were sent home during their their first month of enlistment?,TRUE, +many of the smaller students were too physically small.,That's true---some boys were found to be too small or scared to fight after volunteering for the war. Now be more specific. How many boys were sent home during their their first month of enlistment?,TRUE, +many were too scared to fight.,That's true---some boys were found to be too small or scared to fight after volunteering for the war. Now be more specific. How many boys were sent home during their their first month of enlistment?,TRUE, +they were not able to gain entrance,It's true that they weren't able to enlist. Add another detail from the text to make your response more specific. What prevented some boys from volunteering in the British Army?,TRUE, +many were too young and too scared to fight on the front lines.,That's true---some boys were found to be too small or scared to fight after volunteering for the war. Now be more specific. How many boys were sent home during their their first month of enlistment?,TRUE, +they were not authorized to do so.,It's true that they weren't allowed to enlist. Add another detail from the text to make your response more specific. What prevented some boys from volunteering in the British Army?,TRUE, +they weren't allowed to enter.,It's true that they weren't allowed to enlist. Add another detail from the text to make your response more specific. What prevented some boys from volunteering in the British Army?,TRUE, +they were unable to gain entrance.,It's true that some boys were unable to gain entrance into the British Army. Add another detail to make your response more specific. What was the age requirement for soldiers?,TRUE, +they weren't able to volunteer,It's true that they weren't able to volunteer. Add another detail from the text to make your response more specific. What prevented some boys from volunteering in the British Army?,TRUE, +they weren't allowed to do so though.,It's true that they weren't allowed to enlist. Add another detail from the text to make your response more specific. What prevented some boys from volunteering in the British Army?,TRUE, +they were to young to fight and not very strong.,It's true that some boys were too young and small to fight. Add another detail from the text to make your response more specific. How many underage boys were discharged from the British Army during World War I?,TRUE, +they were not permitted to do so.,It's true that they weren't allowed to enlist. Add another detail from the text to make your response more specific. What prevented some boys from volunteering in the British Army?,TRUE, +they were not able to gain entry,It's true that many underage boys were not able to join the British Army. Add another detail from the text to make your response more specific. What was the age requirement for soldiers?,TRUE, +Several of them did not make it due to their size and age.,That's true---some boys were found to be too small or scared to fight after volunteering for the war. Now be more specific. How many boys were sent home during their their first month of enlistment?,TRUE, +many of them didn't make it due to their size and actual age.,That's true---some boys were found to be too small or scared to fight after volunteering for the war. Now be more specific. How many boys were sent home during their their first month of enlistment?,TRUE, +some were too scared or small to be of any help to the war and could've been with their parents at home.,That's true---some boys were found to be too small or scared to fight after volunteering for the war. Now be more specific. How many boys were sent home during their their first month of enlistment?,TRUE, +they were not allowed to do so.,It's true that they weren't allowed to enlist. Add another detail from the text to make your response more specific. What prevented some boys from volunteering in the British Army?,TRUE, +they were not able to volunteer.,It's true that some boys were not able to volunteer. Add another detail from the text to make your response more specific. What prevented some boys from volunteering in the British Army?,TRUE, +most of the young adults were discharged because they were to scared and small to fight.,That's true---some boys were found to be too small or scared to fight after volunteering for the war. Now be more specific. How many boys were sent home during their their first month of enlistment?,TRUE, +they were not able to join,It's true that they weren't able to join. Add another detail from the text to make your response more specific. What prevented some boys from volunteering in the British Army?,TRUE, +they were not able to be enlisted,It's true that they weren't able to enlist. Add another detail from the text to make your response more specific. What prevented some boys from volunteering in the British Army?,TRUE, +they were unable to volunteer,It's true that some boys were unable to volunteer. Add another detail from the text to make your response more specific. What prevented some boys from volunteering in the British Army?,TRUE, +they were not able to enter.,It's true that many young boys were not able to enter the British Army during WWI. Add another detail to make your response more specific. What was the age requirement for soldiers?,TRUE, +they weren't able to take part,It's true that many young boys weren't able to take part in the war. Add another detail from the text to make your response more specific. What prevented some boys from volunteering in the British Army?,TRUE, +they weren't able to enlist in the military,It's true that they weren't able to enlist in the military. Add another detail from the text to make your response more specific. What prevented some boys from volunteering in the British Army?,TRUE, +they couldn't enlist,It's true that they couldn't enlist. Add another detail from the text to make your response more specific. What prevented some boys from volunteering in the British Army?,TRUE, +they were unable to enlist,It's true that some boys were unable to enlist. Add another detail from the text to make your response more specific. What prevented some boys from volunteering in the British Army?,TRUE, +they weren'tt able to enlist soldiers.,It's true that they weren't able to enlist. Add another detail from the text to make your response more specific. What prevented some boys from volunteering in the British Army?,TRUE, +it was also a forced societal pressure.,It's true that there was societal pressure for boys to enlist. Add another detail to make your response more specific. What was the age requirement for soldiers in the British Army?,FALSE,unhelpful +many also faced pressure to enlist.,It's true that some boys faced pressure to enlist. Add another detail to make your response more specific. What was the age requirement for soldiers?,FALSE,unhelpful +some felt pride for their country and others were pressured to enlist.,"Instead of focusing on reasons why some boys wanted to volunteer during WWI, write about what prevented them from being able to enlist. What prevented some boys from volunteering in the British Army?",TRUE, +in schools children are taught about nationalism and pride for their country.,"Instead of focusing on a reason why some boys wanted to volunteer during WWI, write about what prevented them from being able to enlist. What prevented some boys from volunteering in the British Army?",TRUE, +school curriculum in Britain during that time period emphasized patriotic values and a devotion to country.,"Instead of focusing on a reason why some boys wanted to volunteer during WWI, write about what prevented them from being able to enlist. What prevented some boys from volunteering in the British Army?",TRUE, +nationalistic pride was instilled in them through their studies at school.,"Instead of focusing on a reason why some boys wanted to volunteer during WWI, write about what prevented them from being able to enlist. What prevented some boys from volunteering in the British Army?",TRUE, +these boys had great patriotism for Great Britain.,"Instead of focusing on a reason why some boys wanted to volunteer during WWI, write about what prevented them from being able to enlist. What prevented some boys from volunteering in the British Army?",TRUE, +since they were underage they had to forge documents and give fake names to do so.,"That's something that happened as a result of young boys' interest in joining the army. Since this sentence uses 'but,' write about a contrasting idea about them wanting to enlist instead. What prevented some boys from volunteering in the British Army?",TRUE, +the British government encouraged teachers to teach patriotism.,"Instead of focusing on a reason why some boys wanted to volunteer during WWI, write about what prevented them from being able to enlist. What prevented some boys from volunteering in the British Army?",TRUE, +the boys learned about patriotic values that are taught in their schools.,"Instead of focusing on a reason why some boys wanted to volunteer during WWI, write about what prevented them from being able to enlist. What prevented some boys from volunteering in the British Army?",TRUE, +there was immense pressure asserted by the government and others for men to be patriotic and serve their country.,"Instead of focusing on a reason why some boys wanted to volunteer during WWI, write about what prevented them from being able to enlist. What prevented some boys from volunteering in the British Army?",TRUE, +they were pressured to enlist in their everyday lives.,It's true that some boys faced pressure to enlist. Add another detail to make your response more specific. What was the age requirement for soldiers?,FALSE,unhelpful +the government demanded teachers pressure students to support their country.,"Instead of focusing on a reason why some boys wanted to volunteer during WWI, write about what prevented them from being able to enlist. What prevented some boys from volunteering in the British Army?",TRUE, +propaganda taught in the schools made them feel it was their duty to support the empire.,"Instead of focusing on a reason why some boys wanted to volunteer during WWI, write about what prevented them from being able to enlist. What prevented some boys from volunteering in the British Army?",TRUE, +propaganda and societal pressure that led them to believe that enlisting was a patriotic duty.,"Instead of focusing on reasons why some boys wanted to volunteer during WWI, write about what prevented them from being able to enlist. What prevented some boys from volunteering in the British Army?",TRUE, +propaganda to join the troops was very advertised.,"Instead of focusing on a reason why some boys wanted to volunteer during WWI, write about what prevented them from being able to enlist. What prevented some boys from volunteering in the British Army?",TRUE, +British schools were heavily emphasizing patriotic values at the time.,"It's true that British schools emphasized patriotic values during that time. However, the sentence starts with ""Many young teenage boys wanted to volunteer in the British Army during WWI, but."" This implies a contrasting idea. What prevented these boys from enlisting, despite their desire to do so?",FALSE,verbose +many were able to give fake names and agess when they went to volunteer for service.,"That's something that happened as a result of young boys' interest in joining the army. Since this sentence uses 'but,' write about a contrasting idea about them wanting to enlist instead. What prevented some boys from volunteering in the British Army?",TRUE, +there was a lot of social pressure to be patriotic and do their duty.,"It's true that there was a lot of social pressure to be patriotic and do their duty. However, the sentence starts with ""but"", which indicates that a contrasting idea should follow. The text also mentions that there was an age requirement. Try to incorporate that information into your response.",TRUE, +"they first had to find a way around the legal age requirement by giving fake names, birth certificates and sometime forged documents.","That's something that happened as a result of young boys' interest in joining the army. Since this sentence uses 'but,' write about a contrasting idea about them wanting to enlist instead. What prevented some boys from volunteering in the British Army?",TRUE, +the were devoted to their country and eager to show it.,"It's true that some boys were devoted to their country and eager to show it. The sentence uses 'but', which indicates you should write about something that prevented them from enlisting. What was the age requirement for the British Army?",FALSE,unhelpful +"in their schools, the boys learned patriotic values.","Instead of focusing on a reason why some boys wanted to volunteer during WWI, write about what prevented them from being able to enlist. What prevented some boys from volunteering in the British Army?",TRUE, +the ones who were underage faked their names and ages to get in.,"That's something that happened as a result of young boys' interest in joining the army. Since this sentence uses 'but,' write about a contrasting idea about them wanting to enlist instead. What prevented some boys from volunteering in the British Army?",TRUE, +some joined the army using fake IDs and names.,"That's something that happened as a result of young boys' interest in joining the army. Since this sentence uses 'but,' write about a contrasting idea about them wanting to enlist instead. What prevented some boys from volunteering in the British Army?",TRUE, +many young boys were severely pressured to join the war.,"It's true that many young boys were pressured to join the war. However, the sentence starts with ""but"", which implies a contrasting idea. The text mentions an age requirement that prevented many boys from enlisting. Can you revise your response to reflect this contrast?",FALSE,tone +some of them provided fake names and lied about their real ages in order to enlist.,"That's something that happened as a result of young boys' interest in joining the army. Since this sentence uses 'but,' write about a contrasting idea about them wanting to enlist instead. What prevented some boys from volunteering in the British Army?",TRUE, +related political films were shown and patriotic concerts were held.,"Instead of focusing on a reason why some boys wanted to volunteer during WWI, write about what prevented them from being able to enlist. What prevented some boys from volunteering in the British Army?",TRUE, +"they faced a lot of pressure to enlist in their day-to-day lives through recruitment postcards, political shows, and school curriculum.","It's true that some boys faced pressure to enlist. Since this sentence uses 'but', write about a barrier (or something that made it difficult) some boys faced when wanting to volunteer for the war instead. What prevented some boys from volunteering in the British Army?",FALSE,mixed_message +"these boys found various other ways to join the war, such as fake identities.","That's something that happened as a result of young boys' interest in joining the army. Since this sentence uses 'but,' write about a contrasting idea about them wanting to enlist instead. What prevented some boys from volunteering in the British Army?",TRUE, +they all signed up using fake names and ages.,"That's something that happened as a result of young boys' interest in joining the army. Since this sentence uses 'but,' write about a contrasting idea about them wanting to enlist instead. What prevented some boys from volunteering in the British Army?",TRUE, +they lied about their age in order to enlist.,"That's something that happened as a result of young boys' interest in joining the army. Since this sentence uses 'but,' write about a contrasting idea about them wanting to enlist instead. What prevented some boys from volunteering in the British Army?",TRUE, +school was a requirement for those under 12.,Try clearing your response and starting again. What requirement kept some boys from volunteering in the British Army during World War I? Check that your response only uses information from the text.,TRUE, +the British Army's legal age requirement caused young boys to give fake names and ages to volunteer in the army.,"That's something that happened as a result of young boys' interest in joining the army. Since this sentence uses 'but,' write about a contrasting idea about them wanting to enlist instead. What prevented some boys from volunteering in the British Army?",TRUE, +underage boys found ways to join the army.,It's true that underage boys found ways to join the army. Add another detail to make your response more specific. What was the age requirement for soldiers?,FALSE,unhelpful +"these boys found various other ways to enlist in the British Army, such as using fake identities.","That's something that happened as a result of young boys' interest in joining the army. Since this sentence uses 'but,' write about a contrasting idea about them wanting to enlist instead. What prevented some boys from volunteering in the British Army?",TRUE, +many were already working to support their families.,"It's true that many boys were working to support their families. However, this sentence uses 'but', which indicates a contrasting idea. What requirement prevented some boys from volunteering in the British Army during World War I?",FALSE,unhelpful +"to be able to do this they had to gain fake names and fake ID, this was either helped by support of teachers or parents.","That's something that happened as a result of young boys' interest in joining the army. Since this sentence uses 'but,' write about a contrasting idea about them wanting to enlist instead. What prevented some boys from volunteering in the British Army?",TRUE, +many of them lied about their age in order to get into the service.,"That's something that happened as a result of young boys' interest in joining the army. Since this sentence uses 'but,' write about a contrasting idea about them wanting to enlist instead. What prevented some boys from volunteering in the British Army?",TRUE, +many of them lied about their age to get in the military service.,"That's something that happened as a result of young boys' interest in joining the army. Since this sentence uses 'but,' write about a contrasting idea about them wanting to enlist instead. What prevented some boys from volunteering in the British Army?",TRUE, +"it lead to young men to giving fake names and ages, lying about themselves in order to serve.","That's something that happened as a result of young boys' interest in joining the army. Since this sentence uses 'but,' write about a contrasting idea about them wanting to enlist instead. What prevented some boys from volunteering in the British Army?",TRUE, +they had to sneak in the army by giving their fake names and ages.,"That's something that happened as a result of young boys' interest in joining the army. Since this sentence uses 'but,' write about a contrasting idea about them wanting to enlist instead. What prevented some boys from volunteering in the British Army?",TRUE, +they had to forge paperwork and lie about there ages.,"That's something that happened as a result of young boys' interest in joining the army. Since this sentence uses 'but,' write about a contrasting idea about them wanting to enlist instead. What prevented some boys from volunteering in the British Army?",TRUE, +underage boys found ways to join the army even though they had no legal documentation to show they were 18.,"It's true that some underage boys found ways around the age requirement to join the army. However, the sentence starts with 'but,' which indicates a contrasting idea. The text mentions that many underage boys were sent home. Try focusing on that aspect.",FALSE,bonkers +"the boys wanted to help their country so, the boys took on fake names and many got birth certificate and IDs from supporting teachers and family .","That's something that happened as a result of young boys' interest in joining the army. Since this sentence uses 'but,' write about a contrasting idea about them wanting to enlist instead. What prevented some boys from volunteering in the British Army?",TRUE, +compulsory education for children aged 6 to 12.,Try clearing your response and starting again. What requirement kept some boys from volunteering in the British Army during World War I? Check that your response only uses information from the text.,TRUE, +a lot of underage boys had to find ways to sneak into the British Army by either forging their birth certificates or lying about their age.,"That's something that happened as a result of young boys' interest in joining the army. Since this sentence uses 'but,' write about a contrasting idea about them wanting to enlist instead. What prevented some boys from volunteering in the British Army?",TRUE, +many boys falsified their identities and ages in order to join.,"That's something that happened as a result of young boys' interest in joining the army. Since this sentence uses 'but,' write about a contrasting idea about them wanting to enlist instead. What prevented some boys from volunteering in the British Army?",TRUE, +they had to find a way to sneak in by using either forged documents or lying about their age.,"That's something that happened as a result of young boys' interest in joining the army. Since this sentence uses 'but,' write about a contrasting idea about them wanting to enlist instead. What prevented some boys from volunteering in the British Army?",TRUE, +underage boys found ways to join the army even though they had no legal documentation.,"It's true that some underage boys found ways around the age requirement. However, the sentence starts with 'but,' which indicates a contrasting idea is needed. The text mentions that the army had an age requirement. What was that requirement, and how did it affect underage boys who wanted to enlist?",FALSE,two_directions +"a lot of the time, they had to use fake identification, because the British wouldn't except some of them, but wouldn't check their fake names and ages.","That's something that happened as a result of young boys' interest in joining the army. Since this sentence uses 'but,' write about a contrasting idea about them wanting to enlist instead. What prevented some boys from volunteering in the British Army?",TRUE, +"many were already working to support their families by providing income, child care, or other assistance.","It's true that many young boys were already working to support their families. Since this sentence uses 'but', write about a barrier (or something that made it difficult) some boys faced when wanting to volunteer for the war instead. What prevented some boys from volunteering in the British Army?",FALSE,unhelpful +they needed to arm themselves properly.,"It's true that soldiers needed to be armed properly. However, this response does not address the prompt's request to explain why many young teenage boys who wanted to volunteer in the British Army during WWI were unable to do so. What prevented them from enlisting?",FALSE,verbose +they needed weapons that were properly fitted for the task at hand.,"It's true that soldiers needed properly fitted weapons. Since this sentence uses 'but', write about a barrier (or something that made it difficult) some boys faced when wanting to volunteer for the war instead. What prevented some boys from volunteering in the British Army?",FALSE,mixed_message +the patient's health sometimes ceased this.,Try clearing your response and starting again. What requirement kept some boys from volunteering in the British Army during World War I? Check that your response only uses information from the text.,TRUE, +they pushed the agenda to get poorer children in.,"It's true that some boys wanted to join the British Army during WWI. Since this sentence uses 'but', write about a barrier (or something that made it difficult) some boys faced when wanting to volunteer for the war instead. What prevented some boys from volunteering in the British Army?",FALSE,bonkers +they needed firearms.,"It's true that firearms were used in WWI. However, the sentence asks about why many young teenage boys *wanted* to volunteer but *couldn't*. What was the main obstacle preventing them from enlisting?",FALSE,mixed_message +they pushed the new kids into the group.,"It's true that some boys tried to join the army, but this response does not explain why many young teenage boys were prevented from volunteering. What was the age requirement for the British Army during WWI?",FALSE,unhelpful +Ailments sometimes stopped this.,"It's true that some boys may have had ailments that prevented them from enlisting. However, the sentence asks about why many young teenage boys *wanted* to volunteer but *couldn't*. The text focuses on the age requirement and the consequences of underage enlistment. Try to use information from the text to explain what prevented these boys from enlisting, even if they wanted to.",FALSE,bonkers +many teenage boys had to work to support their families.,"It's true that many boys worked to support their families. However, the sentence starts with ""Many young teenage boys wanted to volunteer in the British Army during WWI, but."" This implies a contrasting idea. What prevented these boys from enlisting, despite their desire to do so?",FALSE,verbose +"because school was mandatory for all children under the age of 12, they could not enlist.","It's true that school was mandatory for children under 12. However, the sentence asks about why many young teenage boys who wanted to volunteer in the British Army during WWI could not. The text states that the army required new recruits to be at least 18. Try to use that information in your response.",FALSE,gives_answer +they didn't want to die.,"It's true that some boys may not have wanted to die. However, the sentence asks you to explain why many young teenage boys were prevented from volunteering in the British Army during WWI. The text explains that there was an age requirement. Can you use that information to complete the sentence?",FALSE,gives_answer +some didn't belong on the front line.,It's true that some boys didn't belong on the front lines. Add another detail to make your response more specific. How many underage boys were discharged from the British Army during World War I?,FALSE,unhelpful +great Britain were seen as enemies and a threat.,"It's true that those who opposed Great Britain were seen as enemies. Since this sentence uses 'but', write about a barrier (or something that made it difficult) some boys faced when wanting to volunteer for the war instead. What prevented some boys from volunteering in the British Army?",FALSE,unhelpful +they had school.,"It's true that many young boys had school, but this response doesn't explain why many young teenage boys were unable to volunteer in the British Army during WWI. What was the age requirement for the British Army?",FALSE,unhelpful +they were not yet 15 years old.,It's true that many boys were too young to enlist. The text mentions a specific age requirement. Can you add that detail to your response?,TRUE, +"nearly 250,000 teenagers would join to call a fight.","It's true that many underage boys tried to join the British Army during WWI. However, your response doesn't explain why many of them were sent home. The text mentions that many underage soldiers were sent home when their parents verified their age, or after admitting their true age. Try to include this information in your response.",FALSE,gives_answer +2.5 million British men eventually volunteered for the wars so that made that young teenager not want to do it no more.,"The text states that 2.5 million British men volunteered. However, the sentence you are completing asks about why many young teenage boys *wanted* to volunteer, but then did not. Your response does not explain why they did not volunteer. Try to use information from the text to explain why many young teenage boys wanted to volunteer, but then did not.",FALSE,verbose \ No newline at end of file diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/data/whales_suboptimal.csv b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/data/whales_so.csv similarity index 83% rename from services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/data/whales_suboptimal.csv rename to services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/data/whales_so.csv index 29de0a06fe7..3adc318d842 100644 --- a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/data/whales_suboptimal.csv +++ b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/data/whales_so.csv @@ -1,188 +1,188 @@ -entry,feedback,optimal_broad,optimal,rule, -they sending drones to gather listening devices to the whale.,"It's true that CETI is using drones. Now be more specific. Why are they sending drones, and what are they gathering?",FALSE,FALSE,two_directions, -they use drones to attach specially-designed listening devices.,It's true that they're using drones and specially-designed listening devices! Now add to your response. How are they using the drones and listening devices to collect more data?,FALSE,TRUE,unhelpful, -the CETI team is working on solutions like creating drones specifically designed to listen to whales.,It's true that CETI is using drones. Now add to your response. What are the drones doing and how are they helping CETI get the data they need?,FALSE,FALSE,two_directions, -they can more effectively find the patterns in the communications of whales.,"Try clearing your response and trying again. Instead of explaining how the data will be used, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -the complexities of the whale clicks can be unraveled.,"Try clearing your response and trying again. Instead of talking about the complexities of whale clicks, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -they have made drones that have attatched specially designed listening devices.,It's true that they're using drones with listening devices! Now add to your response. How are they attaching the listening devices to collect the whale sounds?,TRUE,TRUE,, -they can talk to whales.,"Try clearing your response and trying again. Instead of talking about CETI's mission to communicate with whales, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -researchers hope that the project will help humans engage in dialogue with whales.,"Try clearing your response and trying again. Instead of talking about CETI's mission to communicate with whales, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -"Eventually, it will be possible to decipher the whales' language and converse with them.","Try clearing your response and trying again. Instead of talking about CETI's mission to communicate with whales, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -scientists and researchers can learn more quickly in research.,"Try clearing your response and trying again. Instead of talking about the benefits of collecting more data, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -scientist can study whale communication.,"Try clearing your response and trying again. Instead of talking about what scientists can do with the data, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -"in the future, decoding the whales' language for communication with the creatures is a potential development.","Try clearing your response and trying again. Instead of talking about CETI's mission to communicate with whales, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -humans can engage with species at risk of extinction.,"Try clearing your response and trying again. Instead of talking about CETI's mission to communicate with whales, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -when there on the move they can communicate with the whales,"Try clearing your response and trying again. Instead of talking about how whales communicate, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",FALSE,FALSE,misacknowledgement, -"at present, they possess only thousands of whale recordings, which falls short of what's required to comprehend a language.","Try clearing your response and trying again. Instead of explaining how much data is needed, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -"currently, they have only thousands of whale recordings, which is insufficient for understanding a language.","Try clearing your response and trying again. Instead of explaining how much data is needed, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -People now have a greater understanding of animals.,"Try clearing your response and trying again. Instead of talking about what humans might learn from whales, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -eventually the whales' language can be decoded and used to communicate with the creatures.,"Try clearing your response and trying again. Instead of talking about CETI's mission to communicate with whales, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -they can be able to collect recordings of whales clicking.,"Try clearing your response and trying again. Instead of talking about what CETI can do with the data, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -they use whales and their language.,"Try clearing your response and trying again. Instead of talking about whales and their language, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -"what they are missing are recordings of tens of millions of whales, which they need.","Try clearing your response and trying again. Instead of explaining how much data is needed, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -they can be able to collect data and communicate with whales.,"Try clearing your response and trying again. Instead of talking about CETI's mission to communicate with whales, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -they can hear the voices of these whales that are thousands of feet deep in the water.,"Try clearing your response and trying again. Instead of talking about what CETI can do with the data, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -they can easily identify on what the whales are saying.,"Try clearing your response and trying again. Instead of talking about CETI's mission to communicate with whales, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -they will have to catch a whale because they are always on the move.,"Try clearing your response and trying again. Instead of talking about the challenges of catching whales, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -"presently, they hold only thousands of whale recordings, and this quantity is insufficient for understanding a language.","Try clearing your response and trying again. Instead of explaining how much data is needed, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -efforts to prevent extinction can be expanded through learning as much as possible about whales.,"Try clearing your response and trying again. Instead of talking about CETI's mission to communicate with whales, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -they plan to use LLM to firgure out how whales communicate.,"Try clearing your response and trying again. Instead of talking about the methods CETI is using to understand whale communication, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",FALSE,FALSE,mixed_message, -researchers can effectively translate different whale vibrations and sounds.,"Try clearing your response and trying again. Instead of talking about CETI's mission to communicate with whales, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -"what's missing from their collection are recordings in the tens of millions of whales, which they need.","Try clearing your response and trying again. Instead of explaining how much data is needed, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -"humans will be able to talk to whales, and whales will be able to talk with humans.","Try clearing your response and trying again. Instead of talking about CETI's mission to communicate with whales, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -"their present holdings number in the thousands, but they require tens of millions to meet their needs.","Try clearing your response and trying again. Instead of explaining how much data is needed, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -there will be a clear line of communication.,"Try clearing your response and trying again. Instead of talking about CETI's mission to communicate with whales, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -"what they require are recordings in the tens of millions of whales, which they currently do not have.","Try clearing your response and trying again. Instead of explaining how much data is needed, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -the long-term goal is to decode the language of the whales and use it for communication with the creatures.,"Try clearing your response and trying again. Instead of talking about CETI's mission to communicate with whales, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -"they require whale recordings in the tens of millions, which they do not currently have.","Try clearing your response and trying again. Instead of explaining how much data is needed, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -"their current count is in the thousands, which is far from the tens of millions they require.","Try clearing your response and trying again. Instead of explaining how much data is needed, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -"right now, they are in possession of only thousands of whale recordings, and this number is insufficient for language understanding.","Try clearing your response and trying again. Instead of explaining how much data is needed, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -It is possible to decipher the complexity of the whale clicks.,"Try clearing your response and trying again. Instead of talking about what is possible to do with the data, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -they can communicate with whales better without any misunderstandings.,"Try clearing your response and trying again. Instead of talking about CETI's mission to communicate with whales, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -"they have thus far accumulated only thousands, whereas tens of millions are still required.","Try clearing your response and trying again. Instead of explaining how much data is needed, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -"in the future, the whales' language could be deciphered and utilized to interact with these creatures.","Try clearing your response and trying again. Instead of talking about CETI's mission to communicate with whales, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -they need thousands of more sightings of it.,"Try clearing your response and trying again. Instead of explaining how much data is needed, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -"they currently possess only thousands of whale recordings, which proves inadequate for language understanding.","Try clearing your response and trying again. Instead of explaining how much data is needed, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -"Hopefully, the team will be able to keep coming up with new methods to further its use in learning about whales.","Try clearing your response and trying again. Instead of talking about the future of CETI's research, focus on how CETI is getting their data now. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -they have collected thousands of whale recordings to see a pattern in whale communication.,"Try clearing your response and trying again. Instead of explaining how much data CETI has, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -they only have thousands out of the tens of millions of data points they need.,"Try clearing your response and trying again. Instead of explaining how much data is needed, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -"team has thousands of whale recordings, but to find the patterns in their communication,","Try clearing your response and trying again. Instead of explaining how much data is needed, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -"through the analysis of numerous language samples, an LLM can find patterns that underlie the language's operation.","Try clearing your response and trying again. Instead of explaining how LLMs work, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -"by reviewing a diverse set of language instances, an LLM can map out the rules and patterns it discovers.","Try clearing your response and trying again. Instead of explaining how LLMs work, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -the CETI team has thousands of whale recordings.,"Try clearing your response and trying again. Instead of explaining how much data CETI has, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -they have millions of recordings from when they looked for whales.,"Try clearing your response and trying again. Instead of explaining how much data is needed, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -in order to find a pattern in the recordings they would need to have tens of millions of recordings instead of the thousands.,"Try clearing your response and trying again. Instead of explaining how much data is needed, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -lLMs can enhance their word comprehension in specific contexts through the utilization of a massive dataset.,"Try clearing your response and trying again. Instead of explaining how LLMs work, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -they are often looking at the patterns of whale communication and has over 10 million recordings off of the whales that they have recording devices on.,"Try clearing your response and trying again. Instead of explaining how much data is needed, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -"with access to copious data, lLMs can refine their understanding of a word in specific contexts.","Try clearing your response and trying again. Instead of explaining how LLMs use data, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -AI will work correctly.,"Try clearing your response and trying again. Instead of talking about what AI will do, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -they have thousands of recordings.,"Try clearing your response and trying again. Instead of explaining how much data CETI has, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -they might need tens of millions of recordings to translate the whale clicks.,"Try clearing your response and trying again. Instead of explaining how much data is needed, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -"through the examination of a vast number of language examples, an LLM can uncover patterns governing the language's structure.","Try clearing your response and trying again. Instead of explaining how LLMs work, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -"by harnessing a huge dataset, lLMs can better grasp a word in specific contexts with increased accuracy.","Try clearing your response and trying again. Instead of explaining how LLMs work, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -it does not lead to misunderstanding and false conclusions.,"Try clearing your response and trying again. Instead of explaining a reason CETI needs large amounts of data, focus on how they are getting this data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -misunderstandings and false conclusions can be avoided.,"Try clearing your response and trying again. Instead of explaining a reason CETI needs large amounts of data, focus on how they are getting this data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -they can accurately train the models and won't make mistakes.,"Try clearing your response and trying again. Instead of explaining why CETI needs large amounts of data, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -there are no false conclusions.,"Try clearing your response and trying again. Instead of explaining a reason CETI needs large amounts of data, focus on how they are getting this data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -"The CETI team has thousands of whale recordings, but to find the patterns in their communication, they would likely need tens of millions of recordings.","Try clearing your response and trying again. Instead of explaining how much data is needed, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -data sets that are too small can lead to misunderstandings or false conclusions.,"Try clearing your response and trying again. Instead of explaining a reason CETI needs large amounts of data, focus on how they are getting this data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -a small dataset can lead to misunderstandings and false conclusions.,"Try clearing your response and trying again. Instead of explaining a reason CETI needs large amounts of data, focus on how they are getting this data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -a small dataset could result in misunderstandings and wrong conclusions.,"Try clearing your response and trying again. Instead of explaining a reason CETI needs large amounts of data, focus on how they are getting this data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -they work by analyzing a massive dataset and finding the patterns in it,"Try clearing your response and trying again. Instead of explaining how LLMs work, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -they're now using specially attached drones made just for listening.,"It's true that CETI is using drones. Now add to your response. What are the drones attaching to, and why?",TRUE,TRUE,, -a vast amount of data can assist lLMs in comprehending a word with greater precision in particular contexts.,"Try clearing your response and trying again. Instead of explaining why large amounts of data are needed, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -there are not any misunderstandings and confusions.,"Try clearing your response and trying again. Instead of explaining a reason CETI needs large amounts of data, focus on how they are getting this data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -if it is too small it can lead to misunderstandings and wrong info.,"Try clearing your response and trying again. Instead of explaining a reason CETI needs large amounts of data, focus on how they are getting this data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -small amounts of data could cause misunderstandings.,"Try clearing your response and trying again. Instead of explaining a reason CETI needs large amounts of data, focus on how they are getting this data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -It is unclear what message or information is being conveyed in this text.,"Try clearing your response and trying again. Instead of summarizing the text, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -a complete understanding of them necessitates listening to them in their own setting.,"Try clearing your response and trying again. Instead of talking about the importance of listening to whales, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -they assign a series of numbers to each small piece of language.,"Try clearing your response and trying again. Instead of explaining how LLMs process data, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -a small amount of data may produce wrong results and create misunderstandings.,"Try clearing your response and trying again. Instead of explaining a reason CETI needs large amounts of data, focus on how they are getting this data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -they can find the rules and patterns by looking at many examples.,"Try clearing your response and trying again. Instead of explaining how LLMs work, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -whales are helping.,"Try clearing your response and trying again. Instead of talking about whales helping, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -making whale click language is very difficult.,"Try clearing your response and trying again. Instead of talking about the difficulty of whale communication, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -anything smaller can lead to misunderstandings and false conclusions.,"Try clearing your response and trying again. Instead of explaining a reason CETI needs large amounts of data, focus on how they are getting this data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -a dataset that is too small could lead to misunderstandings and false conclusions.,"Try clearing your response and trying again. Instead of explaining a reason CETI needs large amounts of data, focus on how they are getting this data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -life is sweet like cinammon.,"Try clearing your response and trying again. Instead of talking about a general topic, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",FALSE,TRUE,bonkers, -This input does not represent a meaningful sentence.,"Try clearing your response and trying again. Instead of explaining how much data is needed, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",FALSE,FALSE,misacknowledgement, -"if they use a small dataset, it could cause the researchers to misunderstand the meaning of whale codas.","Try clearing your response and trying again. Instead of explaining a reason CETI needs large amounts of data, focus on how they are getting this data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -it can cost a lot o,"Try clearing your response and trying again. Instead of talking about the cost of collecting data, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",FALSE,TRUE,bonkers, -they can analyze patterns and map out rules for the language by creating large datasets.,"Try clearing your response and trying again. Instead of explaining how LLMs work, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -they're going to,"Try clearing your response and trying again. Instead of talking about what CETI is going to do, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",FALSE,TRUE,bonkers,Match -the more modals the better.,"Try clearing your response and trying again. Instead of talking about the number of modals, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",FALSE,TRUE,bonkers, -they are trying to narrow the amount of formation used to make it more efficient.,"Try clearing your response and trying again. Instead of talking about making the data more efficient, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",FALSE,TRUE,bonkers, -"CETI requires extensive date sets for the successful training of its expansive language models, because large-scale training date can promote the machin","Try clearing your response and trying again. Instead of explaining why large datasets are needed, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -they find it difficult to understand what the whales say.,"Try clearing your response and trying again. Instead of talking about the difficulty of understanding whales, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -whales are intellingent they converse with other whales sjdd,"Try clearing your response and trying again. Instead of talking about whales' intelligence, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -they need to learn easily,"Try clearing your response and trying again. Instead of explaining how LLMs learn, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",FALSE,TRUE,bonkers, -ta da da,"Try clearing your response and trying again. Instead of writing something unrelated, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",FALSE,TRUE,tone, -tey need beter partssss,"Try clearing your response and trying again. Instead of explaining how much data is needed, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",FALSE,TRUE,bonkers, -it is super difficult.,"Try clearing your response and trying again. Instead of explaining the difficulty of collecting data, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -they're attaching drones to whales.,"It's true that CETI is using drones. Now add to your response. What are they attaching to the whales with the drones, and why?",FALSE,FALSE,two_directions, -they use suction cups.,It's true that CETI is using suction cups! Now add to your response. What are they using suction cups to attach?,TRUE,TRUE,, -they use a type of suction cup.,It's true that CETI is using suction cups. Now add to your response. What are they using suction cups to attach?,TRUE,TRUE,, -they are using underwater microphones.,It's true that CETI is using underwater microphones. Now be more specific. How are they using these microphones to collect data?,TRUE,TRUE,, -they came up with unique solutions such as suction cup devices.,"It's true that CETI is using suction cups. Now add to your response. What are they using suction cups to attach, and why?",FALSE,FALSE,two_directions, -they find whales and use specific tools like suction cups.,"It's true that they are finding whales and using suction cups. Now add to your response. What are they using the suction cups to attach, and why are they doing this?",FALSE,FALSE,two_directions, -they attach many suction cups.,It's true that they're using suction cups! Now add to your response. What are they using suction cups to attach?,TRUE,TRUE,, -they are using suction cups.,It's true that CETI is using suction cups. Now add to your response. What are they using suction cups to attach?,TRUE,TRUE,, -"they are using drones, and suction cups.",It's true that CETI is using drones and suction cups! Now add to your response. What are they using the drones and suction cups to attach?,TRUE,TRUE,, -they put suction cups on the whales.,It's true that they're using suction cups! Now add to your response. What are they using suction cups to attach?,TRUE,TRUE,, -they will attach to the whale with a suction cup.,It's true that they are using suction cups! Now add to your response. What are they attaching to the whales with the suction cups?,TRUE,TRUE,, -they attach suction cups to the whale.,It's true that they're using suction cups! Now add to your response. What are they using suction cups to attach?,TRUE,TRUE,, -the team is working on unique solutions to get it.,It's true that CETI is working on unique solutions to get more data. Now be more specific. What kinds of solutions are they using?,TRUE,TRUE,, -they use many different devices to record the sound.,"It's true that CETI is using many different devices. Now be more specific. What kinds of devices are they using, and how are they using them?",FALSE,FALSE,two_directions, -they are utilizing audio recording devices,"It's true that CETI is using audio recording devices. Now be more specific. What kinds of devices are they using, and how are they using them to collect data?",FALSE,FALSE,two_directions, -the team uses some unique methods.,It's true that CETI is using unique methods. Now be more specific. What methods are they using?,TRUE,TRUE,, -they use listening devices to gather data.,It's true that CETI is using listening devices to get more data. Now be more specific. What are the listening devices and how are they using them?,FALSE,FALSE,two_directions, -the CETI team are using devices to listen to whales.,"It's true that CETI is using devices to listen to whales. Now add to your response. What kinds of devices are they using, and how are they using them?",FALSE,FALSE,two_directions, -they are using drones and suction cup.,It's true that CETI is using drones and suction cups! Now add to your response. What are they using the suction cups to attach?,TRUE,TRUE,, -they have unique listening devices.,It's true that CETI is using unique listening devices. Now be more specific. What are these devices and how are they using them?,FALSE,FALSE,two_directions, -the team is finding unique ways to get the data.,You're on the right track. Now add to your response. What unique ways are they using to get the data?,TRUE,TRUE,, -it is planning to get more data.,"Try clearing your response and trying again. Instead of explaining that CETI is planning to get more data, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -they send out many recording devices,"Try clearing your response and trying again. Instead of explaining how CETI is getting data, focus on what types of recording devices they are using and how they are using them. What is the CETI team doing to get the recordings they need?",FALSE,FALSE,two_directions, -Scholars have devised many methods for gathering click recordings.,"Try clearing your response and trying again. Instead of explaining the methods used, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",FALSE,TRUE,, -scientist can use special hearing devices to hear what the whales are saying.,"Try clearing your response and trying again. Instead of talking about CETI's mission to communicate with whales, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",FALSE,FALSE,misacknowledgement, -"To get it, the team is developing original solutions.",It's true that CETI is developing original solutions. Now be more specific. What kinds of solutions are they using?,TRUE,TRUE,, -they have to work hard to find it.,"Try clearing your response and trying again. Instead of explaining the difficulty of collecting data, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",FALSE,FALSE,misacknowledgement, -The CETI team is developing a number of tools to make data collecting easier.,It's true that CETI is working on unique solutions to get more data. Now be more specific. What kinds of solutions are they using?,TRUE,TRUE,, -It intends to obtain further data.,"Try clearing your response and trying again. Instead of explaining that CETI intends to obtain further data, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",FALSE,FALSE,mixed_message, -teams are doing more recordings.,"Try clearing your response and trying again. Instead of explaining how much data is needed, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",FALSE,FALSE,misacknowledgement, -They are developing novel methods to collect data on whales.,It's true that CETI is developing novel methods. Now be more specific. What methods are they using?,TRUE,TRUE,, -They are developing some original solutions.,It's true that CETI is working on unique solutions to get more data. Now be more specific. What kinds of solutions are they using?,TRUE,TRUE,, -Groups are recording more content.,"Try clearing your response and trying again. Instead of explaining how much data is needed, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",FALSE,FALSE,misacknowledgement, -The group is coming up with inventive methods to obtain the information.,"Try clearing your response and trying again. Instead of talking about CETI's methods in general terms, focus on the specific methods they are using to collect whale sounds. What are they doing to get the recordings they need?",FALSE,FALSE,mixed_message, -they have begun to use suction cups.,"It's true that CETI is using suction cups. Now add to your response. What are they using suction cups to attach, and why?",FALSE,FALSE,two_directions, -It wants to gather more information.,"Try clearing your response and trying again. Instead of explaining why CETI needs large amounts of data, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -CETI is working on creating some unique methods to efficiently gather this data.,It's true that CETI is working on unique solutions to get more data. Now be more specific. What kinds of solutions are they using?,TRUE,TRUE,, -they're recruiting the assistance of a drone.,It's true that CETI is using drones. Now add to your response. What are they using the drones for?,TRUE,TRUE,, -"in order to get the data that they need, the CETI team is working on some unique solutions.",It's true that CETI is working on unique solutions to get more data. Now be more specific. What kinds of solutions are they using?,TRUE,TRUE,, -they are recruiting the assistance of a drone.,It's true that CETI is using drones. Now add to your response. What are they using the drones for?,TRUE,TRUE,, -scientists use an abundance of tools to study whale communication.,"Try clearing your response and trying again. Instead of talking about the tools used to study whale communication, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",FALSE,FALSE,mixed_message, -"the CETI team has come up with several unique solutions, like drones.","It's true that CETI is using drones. Now add to your response. What are they using the drones for, and what other solutions are they using?",FALSE,FALSE,two_directions, -they spent most of their times working on unique solutions.,"Try clearing your response and trying again. Instead of explaining what the CETI team is doing, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",FALSE,TRUE,bonkers, -the CETI team is creating unique solutions.,It's true that CETI is working on unique solutions to get more data. Now be more specific. What kinds of solutions are they using?,TRUE,TRUE,, -"the CETI group has devised a variety of distinctive alternatives, such as a drone.","It's true that CETI is using drones. Now add to your response. What are they using the drones for, and what other methods are they using?",FALSE,FALSE,two_directions, -CETI is working on some unique solutions.,It's true that CETI is working on unique solutions to get more data. Now be more specific. What kinds of solutions are they using?,TRUE,TRUE,, -"the CETI group has devised numerous distinctive alternatives, such as a drone.","It's true that CETI is using drones. Now add to your response. What are they using the drones for, and how does that help them get more data?",FALSE,FALSE,two_directions, -how they're getting their data they're studying the whales to see what will help them.,"Try clearing your response and trying again. Instead of talking about studying whales, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",FALSE,TRUE,, -"Innovative remedies, like a drone, are being implemented.","It's true that CETI is using drones. Now add to your response. What are they using the drones for, and what else are they doing to collect data?",FALSE,FALSE,two_directions, -the CETI team is working on unique solution for the problem.,It's true that CETI is working on unique solutions to get more data. Now be more specific. What kinds of solutions are they using?,TRUE,TRUE,, -"the CETI team must collect a large amount of data by tracking and sighting whales to help their dataset be more effective, even though it is incredibly hard to get data on whales.","Try clearing your response and trying again. Instead of explaining the challenges of collecting whale data, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -they are utilizing a drone.,"It's true that CETI is using drones. Now add to your response. Why are they using drones, and what do the drones do to help them collect data?",FALSE,FALSE,two_directions, -the CETI team is working on a better way to catch whale dataset.,"Try clearing your response and trying again. Instead of explaining how CETI is working on a better way, focus on what methods they are using to collect data. What is the CETI team doing to get the recordings they need?",FALSE,FALSE,mixed_message, -often they send researchers to go collect data.,"Try clearing your response and trying again. Instead of talking about how researchers collect data, focus on what methods CETI is using to get the recordings they need. What specific technologies or strategies are they employing?",TRUE,TRUE,, -they spend weeks at a time trying to collect that data,"Try clearing your response and trying again. Instead of explaining how difficult it is to collect data, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",TRUE,TRUE,, -when they use drones they can get it back.,"Try clearing your response and trying again. Instead of explaining how the drones help, focus on what the drones are used for. What is the CETI team doing to get the recordings they need?",FALSE,FALSE,mixed_message, -CETI is constantly coming up with unique solutions to train their LLMs.,"Try clearing your response and trying again. Instead of talking about the solutions CETI is coming up with, focus on what those solutions are. What is the CETI team doing to get the recordings they need?",FALSE,FALSE,mixed_message, -they are finding ways to get more of it.,"Try clearing your response and trying again. Instead of explaining how CETI is getting more data, focus on what methods they are using. What specific actions is the CETI team taking to get the recordings they need?",FALSE,FALSE,mixed_message, -the CETI team is working on unique solutiond such as using drones.,It's true that CETI is working on unique solutions to get more data. Now be more specific. What kinds of solutions are they using?,FALSE,FALSE,gives_answer, -the CETI team is collecting date from various sources.,"Try clearing your response and trying again. Instead of explaining how CETI is getting data, focus on the specific methods they are using. What is the CETI team doing to get the recordings they need?",FALSE,FALSE,mixed_message, -they're engaging the aid of a drone.,It's true that CETI is using drones. Now add to your response. What are they using the drones for?,TRUE,TRUE,, -devices have been made to collect data.,"Try clearing your response and trying again. Instead of explaining what devices are used, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",FALSE,FALSE,mixed_message, -the CETI team is working on some unique solutions for example they are using drone technology.,"It's true that CETI is working on unique solutions and using drone technology. Now add to your response. What are they using the drones for, and what other solutions are they using?",FALSE,FALSE,two_directions, -they use drones and microphones.,It's true that CETI is using drones and microphones. Now add to your response. How are they using these tools to collect more data?,FALSE,FALSE,gives_answer, -they are working on unique solutions by using drones.,"It's true that CETI is working on unique solutions and using drones. Now add to your response. What are they using the drones for, and what other solutions are they using?",FALSE,FALSE,two_directions, -they are using drones to attach to whales.,"It's true that CETI is using drones. Now add to your response. What are they attaching to the whales with the drones, and why are they doing this?",FALSE,FALSE,two_directions, -they are utilizing a drone and acoustic sensors.,It's true that CETI is using drones and acoustic sensors. Now add to your response. How are they using these tools to collect more data?,FALSE,FALSE,gives_answer, -they are utilizing mics thousands of feet beneath the sea.,It's true that CETI is placing listening devices on the seafloor. Now be more specific. Why are they doing this?,TRUE,TRUE,, -"the CETI group is developing one-of-a-kind remedies, including specialized audio equipment on the seaflooor.",It's true that CETI is working on unique solutions to get more data. Now be more specific. What kinds of solutions are they using?,FALSE,TRUE,, -"the CETI team is developing distinctive remedies, such as specialized eavesdropping equipment thousands of feet below.","Try clearing your response and trying again. Instead of talking about the equipment, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",FALSE,FALSE,mixed_message, -the CETI team is working on some unique solutions like listening devices on the ocean floor.,It's true that CETI is placing listening devices on the ocean floor. Now be more specific. Why are they doing this?,TRUE,TRUE,, -they are utilizing a drone and audio recording devices.,It's true that CETI is using drones and audio recording devices. Now add to your response. How are they using these tools to collect more data?,FALSE,FALSE,gives_answer, -CETI is putting listening devices on the seafloor.,It's true that CETI is placing listening devices on the seafloor. Now be more specific. Why are they doing this?,TRUE,TRUE,, -they're putting underwater microphones on the sea floor.,It's true that CETI is placing listening devices on the seafloor. Now be more specific. Why are they doing this?,TRUE,TRUE,, -they're using underwater mics on the bottom of the ocean.,It's true that CETI is placing listening devices on the seafloor. Now be more specific. Why are they doing this?,TRUE,TRUE,, -they're using listening devices that go at the bottom of the ocean.,It's true that CETI is placing listening devices on the seafloor. Now be more specific. Why are they doing this?,TRUE,TRUE,, -CETI is placing listening devices on the bottom of the ocean.,It's true that CETI is placing listening devices on the seafloor. Now be more specific. Why are they doing this?,TRUE,TRUE,, -they use mics that work thousands of feet underwater.,It's true that CETI is using underwater microphones. Now add to your response. How are they using these microphones to collect data?,FALSE,FALSE,gives_answer, -they are employing a drone and audio recorders.,It's true that CETI is using drones and audio recorders. Now add to your response. How are they using the drones and audio recorders to collect more data?,FALSE,FALSE,gives_answer, -they're recording at the bottom of the seafloor.,It's true that CETI is placing listening devices on the seafloor. Now be more specific. Why are they doing this?,TRUE,TRUE,, -they are also placing listening devices on the sea floor.,It's true that CETI is placing listening devices on the seafloor. Now be more specific. Why are they doing this?,TRUE,TRUE,, -they're placing special listening devises on the bottom of the ocean.,It's true that CETI is placing listening devices on the seafloor. Now be more specific. Why are they doing this?,TRUE,TRUE,, -the CETI group is currently developing distinct remedies such as specialized hearing equipment on the bottom of the ocean.,It's true that CETI is working on unique solutions to get more data. Now be more specific. What kinds of solutions are they using?,FALSE,TRUE,, -they're using drones to attach to whales.,"It's true that CETI is using drones. Now add to your response. What are they attaching to the whales, and why?",FALSE,FALSE,two_directions, -Scientists affix microphones to whales.,"It's true that scientists are using microphones. Now add to your response. How are they affixing the microphones to the whales, and what is the purpose of doing so?",FALSE,FALSE,two_directions, -they are placing devices to capture whale clicks.,It's true that CETI is placing devices to capture whale clicks. Now be more specific. What kind of devices are they using and where are they placing them?,FALSE,FALSE,two_directions, -they will attach listening devices to the whales.,It's true that CETI is attaching listening devices. Now add to your response. How are they attaching them and for how long will the devices record?,FALSE,FALSE,two_directions, -scientist have created a device that attaches to the whales.,"It's true that scientists created a device that attaches to whales. Now add to your response. What is the purpose of this device, and how does it help CETI collect data?",FALSE,FALSE,two_directions, \ No newline at end of file +entry,feedback,optimal,rule, +they sending drones to gather listening devices to the whale.,"It's true that CETI is using drones. Now be more specific. Why are they sending drones, and what are they gathering?",FALSE,two_directions, +they use drones to attach specially-designed listening devices.,It's true that they're using drones and specially-designed listening devices! Now add to your response. How are they using the drones and listening devices to collect more data?,FALSE,unhelpful, +the CETI team is working on solutions like creating drones specifically designed to listen to whales.,It's true that CETI is using drones. Now add to your response. What are the drones doing and how are they helping CETI get the data they need?,FALSE,two_directions, +they can more effectively find the patterns in the communications of whales.,"Try clearing your response and trying again. Instead of explaining how the data will be used, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,, +the complexities of the whale clicks can be unraveled.,"Try clearing your response and trying again. Instead of talking about the complexities of whale clicks, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,, +they have made drones that have attatched specially designed listening devices.,It's true that they're using drones with listening devices! Now add to your response. How are they attaching the listening devices to collect the whale sounds?,TRUE,, +they can talk to whales.,"Try clearing your response and trying again. Instead of talking about CETI's mission to communicate with whales, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,, +researchers hope that the project will help humans engage in dialogue with whales.,"Try clearing your response and trying again. Instead of talking about CETI's mission to communicate with whales, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,, +"Eventually, it will be possible to decipher the whales' language and converse with them.","Try clearing your response and trying again. Instead of talking about CETI's mission to communicate with whales, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,, +scientists and researchers can learn more quickly in research.,"Try clearing your response and trying again. Instead of talking about the benefits of collecting more data, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,, +scientist can study whale communication.,"Try clearing your response and trying again. Instead of talking about what scientists can do with the data, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,, +"in the future, decoding the whales' language for communication with the creatures is a potential development.","Try clearing your response and trying again. Instead of talking about CETI's mission to communicate with whales, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,, +humans can engage with species at risk of extinction.,"Try clearing your response and trying again. Instead of talking about CETI's mission to communicate with whales, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,, +when there on the move they can communicate with the whales,"Try clearing your response and trying again. Instead of talking about how whales communicate, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",FALSE,misacknowledgement, +"at present, they possess only thousands of whale recordings, which falls short of what's required to comprehend a language.","Try clearing your response and trying again. Instead of explaining how much data is needed, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",TRUE,, +"currently, they have only thousands of whale recordings, which is insufficient for understanding a language.","Try clearing your response and trying again. Instead of explaining how much data is needed, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",TRUE,, +People now have a greater understanding of animals.,"Try clearing your response and trying again. Instead of talking about what humans might learn from whales, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,, +eventually the whales' language can be decoded and used to communicate with the creatures.,"Try clearing your response and trying again. Instead of talking about CETI's mission to communicate with whales, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,, +they can be able to collect recordings of whales clicking.,"Try clearing your response and trying again. Instead of talking about what CETI can do with the data, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,, +they use whales and their language.,"Try clearing your response and trying again. Instead of talking about whales and their language, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,, +"what they are missing are recordings of tens of millions of whales, which they need.","Try clearing your response and trying again. Instead of explaining how much data is needed, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",TRUE,, +they can be able to collect data and communicate with whales.,"Try clearing your response and trying again. Instead of talking about CETI's mission to communicate with whales, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,, +they can hear the voices of these whales that are thousands of feet deep in the water.,"Try clearing your response and trying again. Instead of talking about what CETI can do with the data, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,, +they can easily identify on what the whales are saying.,"Try clearing your response and trying again. Instead of talking about CETI's mission to communicate with whales, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,, +they will have to catch a whale because they are always on the move.,"Try clearing your response and trying again. Instead of talking about the challenges of catching whales, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,, +"presently, they hold only thousands of whale recordings, and this quantity is insufficient for understanding a language.","Try clearing your response and trying again. Instead of explaining how much data is needed, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",TRUE,, +efforts to prevent extinction can be expanded through learning as much as possible about whales.,"Try clearing your response and trying again. Instead of talking about CETI's mission to communicate with whales, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,, +they plan to use LLM to firgure out how whales communicate.,"Try clearing your response and trying again. Instead of talking about the methods CETI is using to understand whale communication, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",FALSE,mixed_message, +researchers can effectively translate different whale vibrations and sounds.,"Try clearing your response and trying again. Instead of talking about CETI's mission to communicate with whales, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,, +"what's missing from their collection are recordings in the tens of millions of whales, which they need.","Try clearing your response and trying again. Instead of explaining how much data is needed, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",TRUE,, +"humans will be able to talk to whales, and whales will be able to talk with humans.","Try clearing your response and trying again. Instead of talking about CETI's mission to communicate with whales, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,, +"their present holdings number in the thousands, but they require tens of millions to meet their needs.","Try clearing your response and trying again. Instead of explaining how much data is needed, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",TRUE,, +there will be a clear line of communication.,"Try clearing your response and trying again. Instead of talking about CETI's mission to communicate with whales, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,, +"what they require are recordings in the tens of millions of whales, which they currently do not have.","Try clearing your response and trying again. Instead of explaining how much data is needed, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",TRUE,, +the long-term goal is to decode the language of the whales and use it for communication with the creatures.,"Try clearing your response and trying again. Instead of talking about CETI's mission to communicate with whales, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,, +"they require whale recordings in the tens of millions, which they do not currently have.","Try clearing your response and trying again. Instead of explaining how much data is needed, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",TRUE,, +"their current count is in the thousands, which is far from the tens of millions they require.","Try clearing your response and trying again. Instead of explaining how much data is needed, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",TRUE,, +"right now, they are in possession of only thousands of whale recordings, and this number is insufficient for language understanding.","Try clearing your response and trying again. Instead of explaining how much data is needed, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",TRUE,, +It is possible to decipher the complexity of the whale clicks.,"Try clearing your response and trying again. Instead of talking about what is possible to do with the data, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,, +they can communicate with whales better without any misunderstandings.,"Try clearing your response and trying again. Instead of talking about CETI's mission to communicate with whales, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,, +"they have thus far accumulated only thousands, whereas tens of millions are still required.","Try clearing your response and trying again. Instead of explaining how much data is needed, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",TRUE,, +"in the future, the whales' language could be deciphered and utilized to interact with these creatures.","Try clearing your response and trying again. Instead of talking about CETI's mission to communicate with whales, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,, +they need thousands of more sightings of it.,"Try clearing your response and trying again. Instead of explaining how much data is needed, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",TRUE,, +"they currently possess only thousands of whale recordings, which proves inadequate for language understanding.","Try clearing your response and trying again. Instead of explaining how much data is needed, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",TRUE,, +"Hopefully, the team will be able to keep coming up with new methods to further its use in learning about whales.","Try clearing your response and trying again. Instead of talking about the future of CETI's research, focus on how CETI is getting their data now. What is the CETI team doing to get the recordings they need?",TRUE,, +they have collected thousands of whale recordings to see a pattern in whale communication.,"Try clearing your response and trying again. Instead of explaining how much data CETI has, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",TRUE,, +they only have thousands out of the tens of millions of data points they need.,"Try clearing your response and trying again. Instead of explaining how much data is needed, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",TRUE,, +"team has thousands of whale recordings, but to find the patterns in their communication,","Try clearing your response and trying again. Instead of explaining how much data is needed, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",TRUE,, +"through the analysis of numerous language samples, an LLM can find patterns that underlie the language's operation.","Try clearing your response and trying again. Instead of explaining how LLMs work, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,, +"by reviewing a diverse set of language instances, an LLM can map out the rules and patterns it discovers.","Try clearing your response and trying again. Instead of explaining how LLMs work, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,, +the CETI team has thousands of whale recordings.,"Try clearing your response and trying again. Instead of explaining how much data CETI has, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",TRUE,, +they have millions of recordings from when they looked for whales.,"Try clearing your response and trying again. Instead of explaining how much data is needed, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",TRUE,, +in order to find a pattern in the recordings they would need to have tens of millions of recordings instead of the thousands.,"Try clearing your response and trying again. Instead of explaining how much data is needed, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",TRUE,, +lLMs can enhance their word comprehension in specific contexts through the utilization of a massive dataset.,"Try clearing your response and trying again. Instead of explaining how LLMs work, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,, +they are often looking at the patterns of whale communication and has over 10 million recordings off of the whales that they have recording devices on.,"Try clearing your response and trying again. Instead of explaining how much data is needed, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",TRUE,, +"with access to copious data, lLMs can refine their understanding of a word in specific contexts.","Try clearing your response and trying again. Instead of explaining how LLMs use data, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,, +AI will work correctly.,"Try clearing your response and trying again. Instead of talking about what AI will do, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,, +they have thousands of recordings.,"Try clearing your response and trying again. Instead of explaining how much data CETI has, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",TRUE,, +they might need tens of millions of recordings to translate the whale clicks.,"Try clearing your response and trying again. Instead of explaining how much data is needed, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",TRUE,, +"through the examination of a vast number of language examples, an LLM can uncover patterns governing the language's structure.","Try clearing your response and trying again. Instead of explaining how LLMs work, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,, +"by harnessing a huge dataset, lLMs can better grasp a word in specific contexts with increased accuracy.","Try clearing your response and trying again. Instead of explaining how LLMs work, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,, +it does not lead to misunderstanding and false conclusions.,"Try clearing your response and trying again. Instead of explaining a reason CETI needs large amounts of data, focus on how they are getting this data. What is the CETI team doing to get the recordings they need?",TRUE,, +misunderstandings and false conclusions can be avoided.,"Try clearing your response and trying again. Instead of explaining a reason CETI needs large amounts of data, focus on how they are getting this data. What is the CETI team doing to get the recordings they need?",TRUE,, +they can accurately train the models and won't make mistakes.,"Try clearing your response and trying again. Instead of explaining why CETI needs large amounts of data, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,, +there are no false conclusions.,"Try clearing your response and trying again. Instead of explaining a reason CETI needs large amounts of data, focus on how they are getting this data. What is the CETI team doing to get the recordings they need?",TRUE,, +"The CETI team has thousands of whale recordings, but to find the patterns in their communication, they would likely need tens of millions of recordings.","Try clearing your response and trying again. Instead of explaining how much data is needed, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",TRUE,, +data sets that are too small can lead to misunderstandings or false conclusions.,"Try clearing your response and trying again. Instead of explaining a reason CETI needs large amounts of data, focus on how they are getting this data. What is the CETI team doing to get the recordings they need?",TRUE,, +a small dataset can lead to misunderstandings and false conclusions.,"Try clearing your response and trying again. Instead of explaining a reason CETI needs large amounts of data, focus on how they are getting this data. What is the CETI team doing to get the recordings they need?",TRUE,, +a small dataset could result in misunderstandings and wrong conclusions.,"Try clearing your response and trying again. Instead of explaining a reason CETI needs large amounts of data, focus on how they are getting this data. What is the CETI team doing to get the recordings they need?",TRUE,, +they work by analyzing a massive dataset and finding the patterns in it,"Try clearing your response and trying again. Instead of explaining how LLMs work, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,, +they're now using specially attached drones made just for listening.,"It's true that CETI is using drones. Now add to your response. What are the drones attaching to, and why?",TRUE,, +a vast amount of data can assist lLMs in comprehending a word with greater precision in particular contexts.,"Try clearing your response and trying again. Instead of explaining why large amounts of data are needed, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",TRUE,, +there are not any misunderstandings and confusions.,"Try clearing your response and trying again. Instead of explaining a reason CETI needs large amounts of data, focus on how they are getting this data. What is the CETI team doing to get the recordings they need?",TRUE,, +if it is too small it can lead to misunderstandings and wrong info.,"Try clearing your response and trying again. Instead of explaining a reason CETI needs large amounts of data, focus on how they are getting this data. What is the CETI team doing to get the recordings they need?",TRUE,, +small amounts of data could cause misunderstandings.,"Try clearing your response and trying again. Instead of explaining a reason CETI needs large amounts of data, focus on how they are getting this data. What is the CETI team doing to get the recordings they need?",TRUE,, +It is unclear what message or information is being conveyed in this text.,"Try clearing your response and trying again. Instead of summarizing the text, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,, +a complete understanding of them necessitates listening to them in their own setting.,"Try clearing your response and trying again. Instead of talking about the importance of listening to whales, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,, +they assign a series of numbers to each small piece of language.,"Try clearing your response and trying again. Instead of explaining how LLMs process data, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,, +a small amount of data may produce wrong results and create misunderstandings.,"Try clearing your response and trying again. Instead of explaining a reason CETI needs large amounts of data, focus on how they are getting this data. What is the CETI team doing to get the recordings they need?",TRUE,, +they can find the rules and patterns by looking at many examples.,"Try clearing your response and trying again. Instead of explaining how LLMs work, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,, +whales are helping.,"Try clearing your response and trying again. Instead of talking about whales helping, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,, +making whale click language is very difficult.,"Try clearing your response and trying again. Instead of talking about the difficulty of whale communication, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,, +anything smaller can lead to misunderstandings and false conclusions.,"Try clearing your response and trying again. Instead of explaining a reason CETI needs large amounts of data, focus on how they are getting this data. What is the CETI team doing to get the recordings they need?",TRUE,, +a dataset that is too small could lead to misunderstandings and false conclusions.,"Try clearing your response and trying again. Instead of explaining a reason CETI needs large amounts of data, focus on how they are getting this data. What is the CETI team doing to get the recordings they need?",TRUE,, +life is sweet like cinammon.,"Try clearing your response and trying again. Instead of talking about a general topic, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",FALSE,bonkers, +This input does not represent a meaningful sentence.,"Try clearing your response and trying again. Instead of explaining how much data is needed, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",FALSE,misacknowledgement, +"if they use a small dataset, it could cause the researchers to misunderstand the meaning of whale codas.","Try clearing your response and trying again. Instead of explaining a reason CETI needs large amounts of data, focus on how they are getting this data. What is the CETI team doing to get the recordings they need?",TRUE,, +it can cost a lot o,"Try clearing your response and trying again. Instead of talking about the cost of collecting data, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",FALSE,bonkers, +they can analyze patterns and map out rules for the language by creating large datasets.,"Try clearing your response and trying again. Instead of explaining how LLMs work, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,, +they're going to,"Try clearing your response and trying again. Instead of talking about what CETI is going to do, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",FALSE,bonkers,Match +the more modals the better.,"Try clearing your response and trying again. Instead of talking about the number of modals, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",FALSE,bonkers, +they are trying to narrow the amount of formation used to make it more efficient.,"Try clearing your response and trying again. Instead of talking about making the data more efficient, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",FALSE,bonkers, +"CETI requires extensive date sets for the successful training of its expansive language models, because large-scale training date can promote the machin","Try clearing your response and trying again. Instead of explaining why large datasets are needed, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,, +they find it difficult to understand what the whales say.,"Try clearing your response and trying again. Instead of talking about the difficulty of understanding whales, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,, +whales are intellingent they converse with other whales sjdd,"Try clearing your response and trying again. Instead of talking about whales' intelligence, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,, +they need to learn easily,"Try clearing your response and trying again. Instead of explaining how LLMs learn, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",FALSE,bonkers, +ta da da,"Try clearing your response and trying again. Instead of writing something unrelated, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",FALSE,tone, +tey need beter partssss,"Try clearing your response and trying again. Instead of explaining how much data is needed, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",FALSE,bonkers, +it is super difficult.,"Try clearing your response and trying again. Instead of explaining the difficulty of collecting data, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",TRUE,, +they're attaching drones to whales.,"It's true that CETI is using drones. Now add to your response. What are they attaching to the whales with the drones, and why?",FALSE,two_directions, +they use suction cups.,It's true that CETI is using suction cups! Now add to your response. What are they using suction cups to attach?,TRUE,, +they use a type of suction cup.,It's true that CETI is using suction cups. Now add to your response. What are they using suction cups to attach?,TRUE,, +they are using underwater microphones.,It's true that CETI is using underwater microphones. Now be more specific. How are they using these microphones to collect data?,TRUE,, +they came up with unique solutions such as suction cup devices.,"It's true that CETI is using suction cups. Now add to your response. What are they using suction cups to attach, and why?",FALSE,two_directions, +they find whales and use specific tools like suction cups.,"It's true that they are finding whales and using suction cups. Now add to your response. What are they using the suction cups to attach, and why are they doing this?",FALSE,two_directions, +they attach many suction cups.,It's true that they're using suction cups! Now add to your response. What are they using suction cups to attach?,TRUE,, +they are using suction cups.,It's true that CETI is using suction cups. Now add to your response. What are they using suction cups to attach?,TRUE,, +"they are using drones, and suction cups.",It's true that CETI is using drones and suction cups! Now add to your response. What are they using the drones and suction cups to attach?,TRUE,, +they put suction cups on the whales.,It's true that they're using suction cups! Now add to your response. What are they using suction cups to attach?,TRUE,, +they will attach to the whale with a suction cup.,It's true that they are using suction cups! Now add to your response. What are they attaching to the whales with the suction cups?,TRUE,, +they attach suction cups to the whale.,It's true that they're using suction cups! Now add to your response. What are they using suction cups to attach?,TRUE,, +the team is working on unique solutions to get it.,It's true that CETI is working on unique solutions to get more data. Now be more specific. What kinds of solutions are they using?,TRUE,, +they use many different devices to record the sound.,"It's true that CETI is using many different devices. Now be more specific. What kinds of devices are they using, and how are they using them?",FALSE,two_directions, +they are utilizing audio recording devices,"It's true that CETI is using audio recording devices. Now be more specific. What kinds of devices are they using, and how are they using them to collect data?",FALSE,two_directions, +the team uses some unique methods.,It's true that CETI is using unique methods. Now be more specific. What methods are they using?,TRUE,, +they use listening devices to gather data.,It's true that CETI is using listening devices to get more data. Now be more specific. What are the listening devices and how are they using them?,FALSE,two_directions, +the CETI team are using devices to listen to whales.,"It's true that CETI is using devices to listen to whales. Now add to your response. What kinds of devices are they using, and how are they using them?",FALSE,two_directions, +they are using drones and suction cup.,It's true that CETI is using drones and suction cups! Now add to your response. What are they using the suction cups to attach?,TRUE,, +they have unique listening devices.,It's true that CETI is using unique listening devices. Now be more specific. What are these devices and how are they using them?,FALSE,two_directions, +the team is finding unique ways to get the data.,You're on the right track. Now add to your response. What unique ways are they using to get the data?,TRUE,, +it is planning to get more data.,"Try clearing your response and trying again. Instead of explaining that CETI is planning to get more data, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",TRUE,, +they send out many recording devices,"Try clearing your response and trying again. Instead of explaining how CETI is getting data, focus on what types of recording devices they are using and how they are using them. What is the CETI team doing to get the recordings they need?",FALSE,two_directions, +Scholars have devised many methods for gathering click recordings.,"Try clearing your response and trying again. Instead of explaining the methods used, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",FALSE,, +scientist can use special hearing devices to hear what the whales are saying.,"Try clearing your response and trying again. Instead of talking about CETI's mission to communicate with whales, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",FALSE,misacknowledgement, +"To get it, the team is developing original solutions.",It's true that CETI is developing original solutions. Now be more specific. What kinds of solutions are they using?,TRUE,, +they have to work hard to find it.,"Try clearing your response and trying again. Instead of explaining the difficulty of collecting data, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",FALSE,misacknowledgement, +The CETI team is developing a number of tools to make data collecting easier.,It's true that CETI is working on unique solutions to get more data. Now be more specific. What kinds of solutions are they using?,TRUE,, +It intends to obtain further data.,"Try clearing your response and trying again. Instead of explaining that CETI intends to obtain further data, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",FALSE,mixed_message, +teams are doing more recordings.,"Try clearing your response and trying again. Instead of explaining how much data is needed, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",FALSE,misacknowledgement, +They are developing novel methods to collect data on whales.,It's true that CETI is developing novel methods. Now be more specific. What methods are they using?,TRUE,, +They are developing some original solutions.,It's true that CETI is working on unique solutions to get more data. Now be more specific. What kinds of solutions are they using?,TRUE,, +Groups are recording more content.,"Try clearing your response and trying again. Instead of explaining how much data is needed, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",FALSE,misacknowledgement, +The group is coming up with inventive methods to obtain the information.,"Try clearing your response and trying again. Instead of talking about CETI's methods in general terms, focus on the specific methods they are using to collect whale sounds. What are they doing to get the recordings they need?",FALSE,mixed_message, +they have begun to use suction cups.,"It's true that CETI is using suction cups. Now add to your response. What are they using suction cups to attach, and why?",FALSE,two_directions, +It wants to gather more information.,"Try clearing your response and trying again. Instead of explaining why CETI needs large amounts of data, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",TRUE,, +CETI is working on creating some unique methods to efficiently gather this data.,It's true that CETI is working on unique solutions to get more data. Now be more specific. What kinds of solutions are they using?,TRUE,, +they're recruiting the assistance of a drone.,It's true that CETI is using drones. Now add to your response. What are they using the drones for?,TRUE,, +"in order to get the data that they need, the CETI team is working on some unique solutions.",It's true that CETI is working on unique solutions to get more data. Now be more specific. What kinds of solutions are they using?,TRUE,, +they are recruiting the assistance of a drone.,It's true that CETI is using drones. Now add to your response. What are they using the drones for?,TRUE,, +scientists use an abundance of tools to study whale communication.,"Try clearing your response and trying again. Instead of talking about the tools used to study whale communication, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",FALSE,mixed_message, +"the CETI team has come up with several unique solutions, like drones.","It's true that CETI is using drones. Now add to your response. What are they using the drones for, and what other solutions are they using?",FALSE,two_directions, +they spent most of their times working on unique solutions.,"Try clearing your response and trying again. Instead of explaining what the CETI team is doing, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",FALSE,bonkers, +the CETI team is creating unique solutions.,It's true that CETI is working on unique solutions to get more data. Now be more specific. What kinds of solutions are they using?,TRUE,, +"the CETI group has devised a variety of distinctive alternatives, such as a drone.","It's true that CETI is using drones. Now add to your response. What are they using the drones for, and what other methods are they using?",FALSE,two_directions, +CETI is working on some unique solutions.,It's true that CETI is working on unique solutions to get more data. Now be more specific. What kinds of solutions are they using?,TRUE,, +"the CETI group has devised numerous distinctive alternatives, such as a drone.","It's true that CETI is using drones. Now add to your response. What are they using the drones for, and how does that help them get more data?",FALSE,two_directions, +how they're getting their data they're studying the whales to see what will help them.,"Try clearing your response and trying again. Instead of talking about studying whales, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",FALSE,, +"Innovative remedies, like a drone, are being implemented.","It's true that CETI is using drones. Now add to your response. What are they using the drones for, and what else are they doing to collect data?",FALSE,two_directions, +the CETI team is working on unique solution for the problem.,It's true that CETI is working on unique solutions to get more data. Now be more specific. What kinds of solutions are they using?,TRUE,, +"the CETI team must collect a large amount of data by tracking and sighting whales to help their dataset be more effective, even though it is incredibly hard to get data on whales.","Try clearing your response and trying again. Instead of explaining the challenges of collecting whale data, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",TRUE,, +they are utilizing a drone.,"It's true that CETI is using drones. Now add to your response. Why are they using drones, and what do the drones do to help them collect data?",FALSE,two_directions, +the CETI team is working on a better way to catch whale dataset.,"Try clearing your response and trying again. Instead of explaining how CETI is working on a better way, focus on what methods they are using to collect data. What is the CETI team doing to get the recordings they need?",FALSE,mixed_message, +often they send researchers to go collect data.,"Try clearing your response and trying again. Instead of talking about how researchers collect data, focus on what methods CETI is using to get the recordings they need. What specific technologies or strategies are they employing?",TRUE,, +they spend weeks at a time trying to collect that data,"Try clearing your response and trying again. Instead of explaining how difficult it is to collect data, focus on how CETI is getting this data. What is the CETI team doing to get the recordings they need?",TRUE,, +when they use drones they can get it back.,"Try clearing your response and trying again. Instead of explaining how the drones help, focus on what the drones are used for. What is the CETI team doing to get the recordings they need?",FALSE,mixed_message, +CETI is constantly coming up with unique solutions to train their LLMs.,"Try clearing your response and trying again. Instead of talking about the solutions CETI is coming up with, focus on what those solutions are. What is the CETI team doing to get the recordings they need?",FALSE,mixed_message, +they are finding ways to get more of it.,"Try clearing your response and trying again. Instead of explaining how CETI is getting more data, focus on what methods they are using. What specific actions is the CETI team taking to get the recordings they need?",FALSE,mixed_message, +the CETI team is working on unique solutiond such as using drones.,It's true that CETI is working on unique solutions to get more data. Now be more specific. What kinds of solutions are they using?,FALSE,gives_answer, +the CETI team is collecting date from various sources.,"Try clearing your response and trying again. Instead of explaining how CETI is getting data, focus on the specific methods they are using. What is the CETI team doing to get the recordings they need?",FALSE,mixed_message, +they're engaging the aid of a drone.,It's true that CETI is using drones. Now add to your response. What are they using the drones for?,TRUE,, +devices have been made to collect data.,"Try clearing your response and trying again. Instead of explaining what devices are used, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",FALSE,mixed_message, +the CETI team is working on some unique solutions for example they are using drone technology.,"It's true that CETI is working on unique solutions and using drone technology. Now add to your response. What are they using the drones for, and what other solutions are they using?",FALSE,two_directions, +they use drones and microphones.,It's true that CETI is using drones and microphones. Now add to your response. How are they using these tools to collect more data?,FALSE,gives_answer, +they are working on unique solutions by using drones.,"It's true that CETI is working on unique solutions and using drones. Now add to your response. What are they using the drones for, and what other solutions are they using?",FALSE,two_directions, +they are using drones to attach to whales.,"It's true that CETI is using drones. Now add to your response. What are they attaching to the whales with the drones, and why are they doing this?",FALSE,two_directions, +they are utilizing a drone and acoustic sensors.,It's true that CETI is using drones and acoustic sensors. Now add to your response. How are they using these tools to collect more data?,FALSE,gives_answer, +they are utilizing mics thousands of feet beneath the sea.,It's true that CETI is placing listening devices on the seafloor. Now be more specific. Why are they doing this?,TRUE,, +"the CETI group is developing one-of-a-kind remedies, including specialized audio equipment on the seaflooor.",It's true that CETI is working on unique solutions to get more data. Now be more specific. What kinds of solutions are they using?,FALSE,, +"the CETI team is developing distinctive remedies, such as specialized eavesdropping equipment thousands of feet below.","Try clearing your response and trying again. Instead of talking about the equipment, focus on how CETI is getting their data. What is the CETI team doing to get the recordings they need?",FALSE,mixed_message, +the CETI team is working on some unique solutions like listening devices on the ocean floor.,It's true that CETI is placing listening devices on the ocean floor. Now be more specific. Why are they doing this?,TRUE,, +they are utilizing a drone and audio recording devices.,It's true that CETI is using drones and audio recording devices. Now add to your response. How are they using these tools to collect more data?,FALSE,gives_answer, +CETI is putting listening devices on the seafloor.,It's true that CETI is placing listening devices on the seafloor. Now be more specific. Why are they doing this?,TRUE,, +they're putting underwater microphones on the sea floor.,It's true that CETI is placing listening devices on the seafloor. Now be more specific. Why are they doing this?,TRUE,, +they're using underwater mics on the bottom of the ocean.,It's true that CETI is placing listening devices on the seafloor. Now be more specific. Why are they doing this?,TRUE,, +they're using listening devices that go at the bottom of the ocean.,It's true that CETI is placing listening devices on the seafloor. Now be more specific. Why are they doing this?,TRUE,, +CETI is placing listening devices on the bottom of the ocean.,It's true that CETI is placing listening devices on the seafloor. Now be more specific. Why are they doing this?,TRUE,, +they use mics that work thousands of feet underwater.,It's true that CETI is using underwater microphones. Now add to your response. How are they using these microphones to collect data?,FALSE,gives_answer, +they are employing a drone and audio recorders.,It's true that CETI is using drones and audio recorders. Now add to your response. How are they using the drones and audio recorders to collect more data?,FALSE,gives_answer, +they're recording at the bottom of the seafloor.,It's true that CETI is placing listening devices on the seafloor. Now be more specific. Why are they doing this?,TRUE,, +they are also placing listening devices on the sea floor.,It's true that CETI is placing listening devices on the seafloor. Now be more specific. Why are they doing this?,TRUE,, +they're placing special listening devises on the bottom of the ocean.,It's true that CETI is placing listening devices on the seafloor. Now be more specific. Why are they doing this?,TRUE,, +the CETI group is currently developing distinct remedies such as specialized hearing equipment on the bottom of the ocean.,It's true that CETI is working on unique solutions to get more data. Now be more specific. What kinds of solutions are they using?,FALSE,, +they're using drones to attach to whales.,"It's true that CETI is using drones. Now add to your response. What are they attaching to the whales, and why?",FALSE,two_directions, +Scientists affix microphones to whales.,"It's true that scientists are using microphones. Now add to your response. How are they affixing the microphones to the whales, and what is the purpose of doing so?",FALSE,two_directions, +they are placing devices to capture whale clicks.,It's true that CETI is placing devices to capture whale clicks. Now be more specific. What kind of devices are they using and where are they placing them?,FALSE,two_directions, +they will attach listening devices to the whales.,It's true that CETI is attaching listening devices. Now add to your response. How are they attaching them and for how long will the devices record?,FALSE,two_directions, +scientist have created a device that attaches to the whales.,"It's true that scientists created a device that attaches to whales. Now add to your response. What is the purpose of this device, and how does it help CETI collect data?",FALSE,two_directions, \ No newline at end of file diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/data_fetcher.rb b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/data_fetcher.rb index 908a7faa49b..3ffbfa8d337 100644 --- a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/data_fetcher.rb +++ b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/data_fetcher.rb @@ -4,9 +4,11 @@ module Evidence module GenAI module Evaluation class DataFetcher < ApplicationService - Item = Data.define(:entry, :feedback, :optimal_broad, :optimal, :rule) + Item = Data.define(:entry, :feedback, :optimal, :rule) - DEFAULT_FILE = 'whales_suboptimal.csv' + WHALES = 'whales_so.csv' + SOLDIERS = 'underage_soldiers_but.csv' + DEFAULT_FILE = WHALES CSV_FILE_PATH = "#{Evidence::Engine.root}/app/services/evidence/gen_ai/evaluation/data/%s" attr_reader :rule, :optimal @@ -34,7 +36,7 @@ def run Item.new( entry: row['entry'], feedback: row['feedback'], - optimal_broad: ActiveModel::Type::Boolean.new.cast(row['optimal_broad']), + # optimal_broad: ActiveModel::Type::Boolean.new.cast(row['optimal_broad']), optimal: ActiveModel::Type::Boolean.new.cast(row['optimal']), rule: row['rule'] ) From 3b0db29305f2cff8ba8ab8b995543bb1f623f791 Mon Sep 17 00:00:00 2001 From: Daniel Drabik Date: Wed, 22 Jan 2025 12:03:39 -0500 Subject: [PATCH 16/51] Update Verbose checker to be looser. --- .../evidence/gen_ai/evaluation/verbose/checker.rb | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/verbose/checker.rb b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/verbose/checker.rb index bdd19c6a548..a9f9d4bf823 100644 --- a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/verbose/checker.rb +++ b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/verbose/checker.rb @@ -5,16 +5,22 @@ module GenAI module Evaluation module Verbose class Checker < Evaluation::BaseChecker - MAX_SIZE = 250 + MAX_SIZE = 300 MAX_SENTENCES = 4 def run - return suboptimal if feedback.size > MAX_SIZE - return suboptimal if feedback.sentence_count > MAX_SENTENCES + return suboptimal if too_long? optimal end + private def too_long? + return true if feedback.sentence_count > MAX_SENTENCES + + feedback.sentence_count == MAX_SENTENCES && + feedback.size > MAX_SIZE + end + private def suboptimal = Result.new(optimal: false, errors: ['verbose']) end end From 9f24bcd2b0e352120f17c9b2284a3997bb045e68 Mon Sep 17 00:00:00 2001 From: Daniel Drabik Date: Wed, 22 Jan 2025 14:28:18 -0500 Subject: [PATCH 17/51] Add more datasets --- .../evaluation/data/new_haitian_because.csv | 800 ++++++++++++++++++ .../data/underage_soldiers_because.csv | 199 +++++ .../gen_ai/evaluation/data_fetcher.rb | 9 +- .../gen_ai/evaluation/feedback_evaluator.rb | 1 - .../gen_ai/evaluation/tone/checker.rb | 2 +- 5 files changed, 1006 insertions(+), 5 deletions(-) create mode 100644 services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/data/new_haitian_because.csv create mode 100644 services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/data/underage_soldiers_because.csv diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/data/new_haitian_because.csv b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/data/new_haitian_because.csv new file mode 100644 index 00000000000..a2737e3175b --- /dev/null +++ b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/data/new_haitian_because.csv @@ -0,0 +1,800 @@ +entry,feedback,optimal,rule +France unlike Spain had recently abolished savery.,"France did end slavery later, but that isn't a reason Louverture wanted Spain as an ally. Clear your response and start again. This time, write about why Louverture partnered with Spain. How did he think Spain could help him?",TRUE, +France unlike Spain had recently abolished slavery.,"France did end slavery later, but that isn't a reason Louverture wanted Spain as an ally. Clear your response and start again. This time, write about why Louverture partnered with Spain. How did he think Spain could help him?",TRUE, +France unlike Spin had recently abolished slavery.,"France did end slavery later, but that isn't a reason Louverture wanted Spain as an ally. Clear your response and start again. This time, write about why Louverture partnered with Spain. How did he think Spain could help him?",TRUE, +Franc unlike Spain had recently abolished slavery.,"France did end slavery later, but that isn't a reason Louverture wanted Spain as an ally. Clear your response and start again. This time, write about why Louverture partnered with Spain. How did he think Spain could help him?",TRUE, +he required assistance against France's formidable military.,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he needed aid against France's powerful army.,You have the right idea! Now be more specific. What kind of aid did Spain offer to Louverture?,TRUE, +he required assistance against the formidable army of France.,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he needed aid against Frnce's powerful army.,You have the right idea! Now be more specific. What kind of aid did Spain offer Louverture?,TRUE, +he needed a strong partner to help defeat French forces and secure freedom.,You're right that Louverture needed help defeating the French and securing freedom for the enslaved people. Now add more detail about what kind of help Spain offered him.,TRUE, +he needed a strong partner to help defeat Frenc forces and secure freedom.,"You're right that Louverture needed help defeating the French and securing freedom for the enslaved people. Now, be more specific. What did Spain offer that would help him achieve those goals?",TRUE, +he perceived it as a tactical chance to bolster the insurgent movement against the French troops in Saint-Domingue.,That's true! Now be more specific. What did Spain offer Louverture to help him defeat France?,TRUE, +he saw it as a strategic opportunity to strengthen the revolutionary movement against the Fench forces in Saint-Domingue.,That's true! Now be more specific. What did Spain offer Louverture that would strengthen the revolutionary movement?,TRUE, +he perceived it as a tactical chance to enhance the rebellious movement against the French troops in Saint-Domingue.,That's true! Now be more specific. What did Spain offer Louverture to help him defeat France?,TRUE, +he saw it as a strategic opportunity to strengthen he revolutionary movement against he French forces in Saint-Domingue.,That's true! Now be more specific. What did Spain offer Louverture that would strengthen his revolutionary movement?,TRUE, +he viewed it as a tactical chance to bolster the insurgent movement against the French troops in Saint-Domingue.,That's true! Now be more specific. What did Spain offer Louverture to help him defeat France?,TRUE, +he saw it as a strategic move to gain resources.,"You have the right idea! Now be more specific. What resources did Spain offer, and what would these resources help Louverture do?",FALSE,two_directions +he saw it as a strategic opportunity to strengthen the revolutionary movement against the French forces in Saint-Domingue.,That's true! Now be more specific. What resources or support did Spain offer that would strengthen the revolutionary movement against the French?,TRUE, +he saw it as a strategic opportunity to strengthen the revolutionary movement against the French forces in Saint-Dominge.,That's true! Now be more specific. What did Spain offer Louverture that would strengthen the revolutionary movement?,TRUE, +he saw it as a strategic opportunity to strengthen the revolutionary movement against the French forces in Sait-Domingue.,That's true! Now be more specific. What did Spain offer Louverture that would strengthen the revolutionary movement?,TRUE, +he saw it as a strategic opportunity to strengthen the revolutonary movement against the French forces in Saint-Domingue.,That's true! Now be more specific. What did Spain offer Louverture to help him defeat France?,TRUE, +he saw it as a strategic oportunity to strengthen the revolutionary movement against the French forces in Saint-Domingue.,That's true! Now be more specific. What resources or support did Spain offer that would strengthen the revolutionary movement?,TRUE, +he saw it as a strategic opportunity too strengthen the revolutionary movement against the French forces in Saint-Domingue.,That's true! Now be more specific. What did Spain offer Louverture that would strengthen the revolutionary movement?,TRUE, +"he sought assistance in defeating the French forces, who were well-equipped and powerful.",You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +"he sought aid in conquering the French troops, who were well-armed and formidable.","You're right that Louverture wanted to defeat the French, but you need to add more detail about what Spain offered to help him do that. What resources or support did Spain provide?",TRUE, +"he sought aid in conquering the French army, who were well-armed and formidable.","You're right that Louverture needed help defeating the French, but you haven't explained what kind of aid Spain offered. What resources or support did Spain provide to help Louverture fight the French?",TRUE, +"he sought assistance in defeating the Fench forces, who were well-equipped and powerful.",You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he sought assistance to bolster his troops against France.,You have the right idea! Now be more specific. What kind of assistance did Spain offer to Louverture?,TRUE, +"he sought assistnce in defeating the French forces, who were well-equipped and powerful.","You're right that Louverture wanted to defeat the French, but you need to be more specific about what kind of assistance he sought from Spain. What resources or support did Spain offer that would help him fight the French?",TRUE, +he sought suport to strengthen his forces against France.,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he sought the promise of freedom for enslavd individuals.,It's true that Spain promised freedom to enslaved people who joined their army. Now be more specific. How did Louverture think this would help him defeat the French?,FALSE,unhelpful +he sought to strengthen his position against colonial forces.,That's true! Now be more specific. What did Spain offer Louverture to help him defeat France?,TRUE, +it allowed him to disrupt French supply and communication lines.,"That was a result of Louverture's partnership with Spain. Clear your response and start again. This time, write about a reason why Louverture wanted to work with Spain.",TRUE, +it was nice and good.,It's true that Louverture benefited from this partnership. Now be more specific. How did Spain help Louverture and his troops?,TRUE, +Spain was offering his support in his fight against the French colonial forces.,That's true! Now be more specific. What kind of support did Spain offer Louverture?,TRUE, +he sought the promise of freedom for enslaved individuals.,It's true that Spain promised freedom to enslaved people who joined their army. Now be more specific. How did Louverture think this would help him defeat the French?,FALSE,unhelpful +he wanted to win control of several territores.,"That was a result of Louverture's partnership with Spain. Clear your response and start again. This time, write about a reason why Louverture wanted to work with Spain.",TRUE, +Spin was at war with France.,That's true! Now be more specific. How could Spain help Louverture defeat France?,TRUE, +he wanted to win cntrol of several territories.,"That was a result of Louverture's partnership with Spain. Clear your response and start again. This time, write about a reason why Louverture wanted to work with Spain.",TRUE, +Spain was at war with France.,That's true! Now be more specific. How could Spain help Louverture defeat France?,TRUE, +he wanted to win control of several territories.,"That was a result of Louverture's partnership with Spain. Clear your response and start again. This time, write about a reason why Louverture wanted to work with Spain.",TRUE, +his ultimate goal to establish an independent Haiti free from both Frenh and Spanish control.,"That was Louverture's ultimate goal, but this response doesn't explain why he allied with Spain. Reread the text and focus on what Spain offered Louverture that would help him achieve his goal.",TRUE, +Spain was at war with Frane.,That's true! Now be more specific. How could Spain help Louverture defeat France?,TRUE, +his ultimate goal to establish an independent Haiti free from both French and Spanish control.,"That was Louverture's ultimate goal, but this response doesn't explain why he allied with Spain. How did he think an alliance with Spain would help him achieve his goal of independence?",TRUE, +his ultimate goal to establish an independent Haiti free from both French and Sanish control.,"That was Louverture's ultimate goal, but this response doesn't explain why he allied with Spain. Reread the text and focus on what Spain offered Louverture that would help him achieve his goal.",TRUE, +it allowed him to disrupt Frech supply and communication lines.,"That was a result of Louverture's partnership with Spain. Clear your response and start again. This time, write about a reason why Louverture wanted to work with Spain.",TRUE, +primarily as a strategic move during the Haitian Revoluton.,"That's true, it was a strategic move. However, to make your response more complete, explain what resources or support Spain offered that made it a strategically beneficial alliance for Louverture.",TRUE, +primarily as a strategic move during the Haitian Revolution.,"That's true, it was a strategic move. However, to make your response more complete, explain what resources or support Spain offered that made it a strategically beneficial alliance for Louverture.",TRUE, +his ultimate goal to establish an independent Haiti free from both French and Spanish cntrol.,"That's true, but that wasn't his reason for allying with Spain. The text explains why he allied with Spain in 1793. What did Spain offer that would help him fight the French?",TRUE, +Spain was offering his support in his fight against the Fench colonial forces.,That's true! Now be more specific. What kind of support did Spain offer Louverture?,TRUE, +Spain was offering his support in his figh against the French colonial forces.,That's true! Now be more specific. What kind of support did Spain offer Louverture?,TRUE, +Spain was offering his support in his fight against he French colonial forces.,That's true! Now be more specific. What kind of support did Spain offer Louverture?,TRUE, +Spin was offering his support in his fight against the French colonial forces.,That's true! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +Spain was offering his suport in his fight against the French colonial forces.,That's true! Now be more specific. What kind of support did Spain offer Louverture?,TRUE, +"Spain assistance, Loverture soon won control several cities, ports, and territories.","That was a result of Louverture's partnership with Spain. Clear your response and start again. This time, write about a reason why Louverture wanted to work with Spain.",TRUE, +"Spain opposed the French government, which had refused to end slaver in its colonies.",That's true! Now be more specific. How could Spain help Louverture defeat France?,TRUE, +Spin would win the war.,That's incorrect. The text describes why Toussaint Louverture formed an alliance with Spain. Reread the text and focus on what Spain offered Louverture to help him fight the French.,FALSE,tone +"Spain opposed the French government, which had refused to end slavery in its colonies.","That's true, but it doesn't explain why Louverture formed an alliance with Spain. The text explains what Spain offered Louverture in exchange for his alliance. Can you use that information to answer the question?",FALSE,tone +Spain would win the war.,"That's true that Spain wanted to win the war, but that's not why Louverture formed an alliance with them. Try again. Think about what Louverture needed to fight the French, and how Spain could help him.",FALSE,tone +"Spain assistance, Louverture soon won control several cities, ports, and territories.","That's a result of Louverture's partnership with Spain. Clear your response and start again. This time, write about a reason why Louverture wanted to work with Spain.",TRUE, +"Spain assisance, Louverture soon won control several cities, ports, and territories.","That was a result of Louverture's partnership with Spain. Clear your response and start again. This time, write about a reason why Louverture wanted to work with Spain.",TRUE, +"Spain opposed the Frenh government, which had refused to end slavery in its colonies.","That's true, but it doesn't explain why Louverture formed an alliance with Spain. The text explains what Spain offered Louverture in exchange for his alliance. Can you use that information to answer the question?",FALSE,tone +"Spai opposed the French government, which had refused to end slavery in its colonies.","That's true, but it doesn't explain why Louverture formed an alliance with Spain. The text explains what Spain offered Louverture in exchange for his alliance. Can you use that information to answer the question?",FALSE,tone +"Spain assistance, Louverture soon won contol several cities, ports, and territories.","That's a result of Louverture's partnership with Spain. Clear your response and start again. This time, write about a reason why Louverture wanted to work with Spain.",TRUE, +"Span, along with Great Britain, opposed the French Revolutionary government.","That's true, but this response doesn't explain why Toussaint Louverture formed an alliance with Spain. The text focuses on Louverture's motivations. Try again, focusing on what Spain offered Louverture to help him fight the French.",TRUE, +"Span assistance, Louverture soon won control several cities, ports, and territories.",That describes what happened after Louverture allied with Spain. The question asks why he formed the alliance in the first place. What did Spain offer that made Louverture want to work with them?,TRUE, +"Spain assistance, Louverture soon won control several cities, ports, and trritories.",That describes what happened after Louverture allied with Spain. Try again. Explain why Louverture decided to make an alliance with Spain in the first place. What did Spain offer him?,TRUE, +"Spain, along with Great Britain, opposed the French Revolutionary government.","That's true, but this response doesn't explain why Toussaint Louverture formed an alliance with Spain. The text focuses on Louverture's reasons for allying with Spain. Try to focus on what Spain offered Louverture to help him in his fight against France.",FALSE,verbose +"Spain, along with Great Britain, opposed the Frenc Revolutionary government.","That's true, but this response doesn't explain why Louverture formed an alliance with Spain. The text focuses on Louverture's reasons for allying with Spain. Try again, focusing on what Spain offered Louverture to help him fight the French.",FALSE,verbose +Spai offered support for his fight against French colonial forces.,It's true that Spain supported Louverture's fight against France. Now be more specific. What kind of support did Spain offer?,TRUE, +the alliance boosted Louverture's reputation as a leader to join Spanish army in 1974.,"That's true, but it's not the reason Louverture formed the alliance. The text explains why Louverture chose to work with Spain. Try again, focusing on what Spain offered Louverture to help him fight the French.",FALSE,verbose +"these became new areas from which he could launch surprise attacks, disrupting French supplies.","That was a result of Louverture's partnership with Spain. Clear your response and start again. This time, write about a reason why Louverture wanted to work with Spain.",TRUE, +the allance boosted Louverture's reputation as a leader to join Spanish army in 1974.,"That's true, but it's not the reason Louverture formed an alliance with Spain in 1793. The text explains why he chose to ally with Spain at that time. Try to find that information in the text and use it to answer the question.",FALSE,verbose +the alliance boosted Louverture's reputaton as a leader to join Spanish army in 1974.,"That's true, but it's not the reason Louverture formed an alliance with Spain in 1793. The text explains why he chose to ally with Spain at that time. Try to find that information in the text and use it to answer the question.",FALSE,verbose +The new areas where Louvertue soon won control of several important places.,"That's a result of Louverture's partnership with Spain. Clear your response and start again. This time, write about a reason why Louverture wanted to work with Spain.",TRUE, +the alliance boosted Louverture's reputation as a leader to join Spaish army in 1974.,"That's true, but it's not the reason Louverture formed an alliance with Spain in 1793. The text explains why he chose to ally with Spain at that time. Try to find that information in the text and use it to answer the question.",FALSE,verbose +the alliance boosted Louvertur's reputation as a leader to join Spanish army in 1974.,"That's true, but it's not the reason Louverture formed the alliance. The text explains why Louverture chose to work with Spain. Try again, focusing on what Spain offered Louverture to help him fight the French.",FALSE,verbose +the needed ther help to fight the strong French forces and free the enslaved people.,You're right that Louverture needed help fighting the French and freeing enslaved people. Can you be more specific about what kind of help Spain offered him?,FALSE,tone +"these became new areas from which he could launch surprise attacks, disrupting Frech supplies.","That was a result of Louverture's partnership with Spain. Clear your response and start again. This time, write about a reason why Louverture wanted to work with Spain.",TRUE, +The new areas where Louverture soon won control of several important places.,"That describes a result of Louverture's alliance with Spain, not a reason for it. Why did Louverture want to partner with Spain? Look for information in the text that explains his reasons for forming the alliance.",FALSE,verbose +"This alliance enabled Louverture to achieve significant military success, allowing him to capture key cities and disrupt French supply lines.","That's a result of Louverture's partnership with Spain. Clear your response and start again. This time, write about a reason why Louverture wanted to work with Spain.",TRUE, +"This alliance enabled Louvrture to achieve significant military success, allowing him to capture key cities and disrupt French supply lines.","That's a result of Louverture's partnership with Spain. Clear your response and start again. This time, write about a reason why Louverture wanted to work with Spain.",TRUE, +"This allance enabled Louverture to achieve significant military success, allowing him to capture key cities and disrupt French supply lines.","That describes the results of the alliance, not the reasons for it. Why did Louverture want to partner with Spain in the first place? What did Spain offer him?",FALSE,two_directions +"This alliance enabled Louverture to achieve significant military success, allowing him to capture key cities and disrupt Frenc supply lines.","That's a result of Louverture's partnership with Spain. Clear your response and start again. This time, write about a reason why Louverture wanted to work with Spain.",TRUE, +The new areas where Louverture soon won cotrol of several important places.,"That describes a result of Louverture's alliance with Spain, not a reason for it. Why did Louverture want to partner with Spain? Look for information in the text that explains his reasons.",FALSE,verbose +They promised freedom to any enslavd person.,It's true that Spain promised freedom to enslaved people who joined their army. Now be more specific. What did Louverture hope to achieve by allying with Spain?,FALSE,gives_answer +They promised freedom to any enslaved person.,It's true that Spain promised freedom to enslaved people who joined their army. Now be more specific. How did Louverture think this promise would help him defeat the French?,FALSE,gives_answer +"to quickly gain control of many important cities, ports and territories.","That was a result of Louverture's partnership with Spain. Clear your response and start again. This time, write about a reason why Louverture wanted to work with Spain.",TRUE, +"to quickly gain control of many important cities, ports and territores.","That was a result of Louverture's partnership with Spain. Clear your response and start again. This time, write about a reason why Louverture wanted to work with Spain.",TRUE, +Spain wanted to regain control of the other half of Hispaniola that it lost to Fance.,"That's a reason why Spain wanted to form a partnership. Clear your response and start again. This time, write about a reason why Louverture wanted to build a relationship.",TRUE, +Spain wanted to regain control of the other half of Hispaniola that it lost to France.,"That's a reason why Spain wanted to form a partnership. Clear your response and start again. This time, write about a reason why Louverture wanted to build a relationship.",TRUE, +"to quickly gain contol of many important cities, ports and territories.","That was a result of Louverture's partnership with Spain. Clear your response and start again. This time, write about a reason why Louverture wanted to work with Spain.",TRUE, +both he and Spain needed help in their struggles with France.,That's true! Now be more specific. What did Spain offer Louverture to help him defeat France?,TRUE, +Spain wanted to regain control of the other half of Hipaniola that it lost to France.,"That's a reason why Spain wanted to form a partnership. Clear your response and start again. This time, write about a reason why Louverture wanted to build a relationship.",TRUE, +Both he and Spain required assistance in their battles against France.,That's true! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +Spain wanted to regain cotrol of the other half of Hispaniola that it lost to France.,"That's a reason why Spain wanted to form a partnership. Clear your response and start again. This time, write about a reason why Louverture wanted to build a relationship.",TRUE, +both he and Spain required assistance in their battles against France.,That's true! Now be more specific. What kind of assistance did Spain offer Louverture?,TRUE, +which still upheld the brutal system of slavry.,"That's true, but it doesn't explain why Louverture formed an alliance with Spain. The text mentions several reasons why Louverture chose to ally with Spain. Try rereading the text and focusing on what Spain offered Louverture and his army.",FALSE,verbose +Span wanted to regain control of the other half of Hispaniola that it lost to France.,"That's a reason why Spain wanted to form a partnership. Clear your response and start again. This time, write about a reason why Louverture wanted to build a relationship.",TRUE, +alignment of alliace improved trade.,That's not quite right. The text explains that Toussaint Louverture formed an alliance with Spain for military reasons. Try reading the text again and focusing on why Louverture needed Spain's help.,FALSE, +to the allance boosted Louverture's reputation as a leader to join the Spanish army in 1974.,"That's true, but it's not the reason Louverture formed an alliance with Spain in 1793. The text explains why he chose to ally with Spain at that time. Try to find that information in the text and use it to answer the question.",FALSE,verbose +to the alliance boosted Louverture's reputatin as a leader to join the Spanish army in 1974.,"That's true, but it's not the reason Louverture formed the alliance. The text explains why Louverture chose to ally with Spain. Try again, focusing on what Spain offered Louverture to help him fight the French.",FALSE,verbose +to the alliance boosted Louerture's reputation as a leader to join the Spanish army in 1974.,"That's true, but it's not the reason Louverture formed the alliance in 1793. The text explains why he allied with Spain. Try again, focusing on what Spain offered Louverture to help him fight the French.",TRUE, +both he and Spain needed help in ther struggles with France.,That's true! Now be more specific. What did Spain offer Louverture to help him defeat France?,TRUE, +alignment of alliance improved trade.,That's not quite right. The text doesn't mention trade as a reason for the alliance. Reread the text and focus on why Louverture needed Spain's help to fight the French.,FALSE, +which still upheld the brutal system of slavery.,"That's true, but it doesn't explain why Louverture formed an alliance with Spain. The text mentions several reasons why Louverture chose to ally with Spain. Try rereading the text and focusing on what Spain offered Louverture and his army.",FALSE,verbose +to the alliance boosted Louverture's reputation as a leader to join the Spaish army in 1974.,"That's true, but it's not the reason Louverture formed the alliance in 1793. The text explains why he allied with Spain. Try again, focusing on what Spain offered Louverture to help him fight France.",FALSE,verbose +to the alliance boosted Louverture's reputation as a leader to join the Spanish army in 1974.,"That's true, but it's not the reason Louverture formed the alliance in 1793. The text explains why he allied with Spain. Try again, focusing on what Spain offered Louverture to help him fight the French.",FALSE,verbose +both he adn Spain needed help in their struggles with France.,That's true! Now be more specific. What did Spain offer Louverture to help him defeat France?,TRUE, +greater force was needed to fight the French.,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +both he and Span needed help in their struggles with France.,That's true! Now be more specific. What did Spain offer Louverture to help him defeat France?,TRUE, +greater force was needed to fight the Fench.,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +greater force was needed too fight the French.,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +greater force was needed to fight he French.,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +more powerful effort was required to combat the French.,You're right that a powerful effort was needed to defeat the French. Now add more information. What did Spain offer Louverture to help him in this effort?,TRUE, +more powerful strength was necessary to combat the French.,You're right that Louverture needed more strength to fight the French. Now be more specific. What did Spain offer to help him gain that strength?,TRUE, +more power was necessary to combat the French.,You're right that Louverture needed more power to fight the French. Can you be more specific about what kind of support he received from Spain to help him gain that power?,FALSE,tone +both he and Spain needed help in their struggles with Frnce.,That's true! Now be more specific. What kind of help did Spain offer Louverture?,TRUE, +greater force was needed to fght the French.,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he felt it would help him in his quest to abolish slavery.,It's true that Louverture wanted to end slavery. Now add more information. How did he think a partnership with Spain would help him achieve this goal?,TRUE, +he felt it would help him in his quest to abolish slaver.,It's true that Louverture wanted to end slavery. Now add more information. How did he think a partnership with Spain would help him achieve this goal?,TRUE, +he lacked the power to defeat the well-equipped Fench army without help.,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he lacked the power to defeat the well-equipped French army without help.,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he needed allies in his war with France.,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he required comrades to combat the French military.,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he needed allies to fight the French army.,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he needed allies too fight the French army.,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he required supporters to combat the French military.,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he needed allies in his war with Franc.,You have the right idea! Now be more specific. What kind of support did Spain offer Louverture?,TRUE, +he needed help defeating France in order to end slavery.,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he required comrades to battle the French military.,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he needed allies to figt the French army.,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he needed allies to fight he French army.,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he needed allies to fight the Fench army.,You have the right idea! Now be more specific. What kind of support did Spain offer Louverture?,TRUE, +he needed help defeating Franc in order to end slavery.,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he needed help defeating France in order too end slavery.,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he needed help from the Spanish in order too fight the French.,You have the right idea! Now be more specific. What kind of help did Spain offer Louverture?,TRUE, +he needed help from he Spanish in order to fight he French.,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he needed help defeating France in order to end savery.,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he needed help from the Spanish in order to fight the French.,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he needed help from the Spanish in order to fight the Frech.,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he needed help from the Spanish in order to figt the French.,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he required assistance in conquering France to put an end to slavery.,"You're right that Louverture wanted to end slavery, and that he needed help defeating France. Now add more information. What did Spain offer to help him achieve this goal?",TRUE, +he needed help from the Spanih in order to fight the French.,You have the right idea! Now be more specific. What kind of help did Spain offer Louverture?,TRUE, +he required assistance in overcoming France to put an end to slavery.,"You're right that Louverture wanted to end slavery, and that he needed help defeating France. Now add more information. What did Spain offer to help him achieve these goals?",TRUE, +he needed help form the Spanish in order to fight the French.,You have the right idea! Now be more specific. What kind of help did Spain offer Louverture?,TRUE, +he needed help if he was going to defeat the French.,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he needed help if he was going to defeat the Frenh.,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he needed help to control military targets.,"You have the right idea! Now be more specific. What kind of help did Louverture need from Spain, and what would that help him achieve?",FALSE,two_directions +he required assistance in achieving liberty from France and terminating enslavement.,It's true that Louverture wanted freedom from France and an end to slavery. Now add more information. What did Spain offer to help Louverture achieve these goals?,TRUE, +he required assistance to attain liberty from France and abolish enslavement.,It's true that Louverture wanted to end slavery and defeat France. Now add more information. What did Spain offer to help Louverture and his troops?,TRUE, +he needed help to gain freedom from France and put an end to slavery.,You're right that Louverture wanted freedom from France and an end to slavery. Now add more information. What did Spain offer to help Louverture achieve these goals?,TRUE, +he needed help to cntrol military targets.,"You have the right idea! Now be more specific. What kind of help did Louverture need from Spain, and what would that help him achieve?",FALSE,two_directions +he needed help to gain freedom from France adn put an end to slavery.,It's true that Louverture wanted freedom from France and an end to slavery. Now add more information. What did Spain offer to help Louverture achieve these goals?,TRUE, +he needed help to gain freedom form France and put an end to slavery.,It's true that Louverture wanted to end slavery and gain freedom from France. Now add more information. What did Spain offer to help Louverture and his troops?,TRUE, +he needed more troops to take on Frane.,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he needed help to gain freedom from France and put an end to slavey.,You're right that Louverture wanted freedom from France and an end to slavery. Now add more information. What did Spain offer to help Louverture achieve these goals?,TRUE, +he needed help too gain freedom from France and put an end too slavery.,You're right that Louverture wanted freedom from France and an end to slavery. Now add more information. What did Spain offer to help Louverture achieve these goals?,TRUE, +he required assistance to obtain liberty from France and terminate enslavement.,It's true that Louverture wanted freedom from France and an end to slavery. Now add more information. What did Spain offer to help Louverture achieve these goals?,TRUE, +he needed help to gain freedom from Fance and put an end to slavery.,You're right that Louverture wanted freedom from France and an end to slavery. Now add more information. What did Spain offer to help Louverture achieve these goals?,TRUE, +he needed more troops to take on France.,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,FALSE,unhelpful +he needed support to rebel against France.,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he needed supplies and support.,You have the right idea! Now add another detail. What would these supplies and support help Louverture do?,TRUE, +he needed support to rebel against Frane.,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he wanted to be able to fight France.,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he needed the sheer number of troops Sain had to be able to defeat France.,"You have the right idea that Louverture needed help from Spain to defeat France. However, the text states that Spain needed more troops, not Louverture. What did Spain offer Louverture to help him fight the French?",TRUE, +he needed the Spnish military to help carry out attacks on the French supply and communications.,"That was a result of Louverture's partnership with Spain. Clear your response and start again. This time, write about a reason why Louverture wanted to work with Spain.",TRUE, +he needed to defeat Franc.,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he needed the sheer number of troops Spain had to be able to defeat France.,"You're right that Louverture needed help defeating France, but the text says Spain needed more troops, not Louverture. What did Spain offer Louverture to help him fight the French?",TRUE, +he needed the sheer number of troops Spain had to be able to defeat Frane.,"You have the right idea! Spain did have a large army, but the text mentions something else that Spain offered Louverture. What resources did Spain offer to help Louverture fight the French?",FALSE,misacknowledgement +he needed the Spanish military to help carry out attacks on the French supply and communications.,"That was a result of Louverture's partnership with Spain. Clear your response and start again. This time, write about a reason why Louverture wanted to work with Spain.",TRUE, +he needed to defeat France.,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he wanted to be powerful enough to defeat France.,You have the right idea! Now be more specific. What did Spain offer Louverture to help him defeat France?,TRUE, +he wanted to be powerful enough to defeat Frane.,You have the right idea! Now be more specific. What did Spain offer Louverture to help him defeat France?,TRUE, +he wanted to be able to fiht France.,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he needed the Spanish military to help carry out attacks on the Fench supply and communications.,"That was a result of Louverture's partnership with Spain. Clear your response and start again. This time, write about a reason why Louverture wanted to work with Spain.",TRUE, +he aimed to conquer France and abolish slavery in Saint-Domingue.,That's true! Now be more specific. How could Spain help Louverture defeat France and abolish slavery?,FALSE,misacknowledgement +he aimed to conquer France and abolish servitude on Saint-Domingue.,That's true! Now be more specific. How could Spain help Louverture defeat France and abolish slavery?,FALSE,affirms_incorrect +he needed the Spanish military to help carry out attacks on the French supply and communicatios.,"That was a result of Louverture's partnership with Spain. Clear your response and start again. This time, write about a reason why Louverture wanted to work with Spain.",TRUE, +he wanted to defeat France and end slavery on Saint-Domingue.,"You're right that Louverture wanted to defeat France and end slavery. To make your answer better, explain how he thought an alliance with Spain would help him achieve those goals. What did Spain offer that would help him fight France?",FALSE,tone +he wanted to be able to fight Frane.,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he aimed to conquer France and abolish slavery on Saint-Domingue.,That's true! Now be more specific. How could Spain help Louverture defeat France and abolish slavery?,FALSE,affirms_incorrect +he desired to conquer France and abolish slavery on Saint-Domingue.,That's true! Now be more specific. How could Spain help Louverture defeat France and abolish slavery?,FALSE,affirms_incorrect +he wanted to defeat France and end slavery on Sint-Domingue.,You're right that Louverture wanted to defeat France and end slavery. Now add more information. How did he think a partnership with Spain would help him achieve these goals?,TRUE, +he wanted too defeat France with their help.,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he wanted to defeat France and free his people.,You have the right idea! Now be more specific. How did he think a partnership with Spain would help him achieve this goal?,TRUE, +he wanted to defeat France adn end slavery on Saint-Domingue.,"You're right that Louverture wanted to defeat France and end slavery. To make your answer better, explain how an alliance with Spain would help him achieve those goals. What did Spain offer that would help him fight France?",FALSE,tone +he desired to conquer France with their assistance.,That's true! Now be more specific. What did Spain offer Louverture to help him defeat France?,TRUE, +he wanted to defeat France with their help.,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he wanted to defeat France and free his peopl.,You have the right idea! Now be more specific. How did Spain help Louverture and his troops?,TRUE, +he aimed to conquer France with their assistance.,That's true! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he wanted to defeat Franc with their help.,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he wanted to defeat France with ther help.,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he wanted too defeat France and end slavery on Saint-Domingue.,It's true that Louverture wanted to defeat France and end slavery. Now add more information. How did he think a partnership with Spain would help him achieve these goals?,TRUE, +he wanted to end slavry by defeating France.,You have the right idea! Now be more specific. How did he think a partnership with Spain would help him achieve this goal?,TRUE, +he wanted to end slavery by defeating Frane.,You have the right idea! Now be more specific. How did he think a partnership with Spain would help him achieve this goal?,TRUE, +he wanted to defeat Frane and end slavery on Saint-Domingue.,"You're right that Louverture wanted to defeat France and end slavery. To make your answer better, explain how he thought an alliance with Spain would help him achieve those goals. What did Spain offer that would help him fight France?",FALSE,tone +he wanted to end slaery.,It's true that Louverture wanted to end slavery. Now add more information. How did he think a partnership with Spain would help him achieve this goal?,TRUE, +he wanted to end slavery.,It's true that Louverture was committed to ending slavery. Now add more information. How did he think a partnership with Spain would help him achieve this goal?,TRUE, +he wanted to defeat Franc and free his people.,You have the right idea! Now be more specific. How did Spain help Louverture and his troops?,TRUE, +he wanted to defeat France and end slavery on Saint-Dmingue.,"You're right that Louverture wanted to defeat France and end slavery. To make your answer better, explain how he thought an alliance with Spain would help him achieve those goals. What did Spain offer that would help him fight France?",FALSE,tone +he wanted to end slavery by defeating France.,It's true that Louverture wanted to end slavery. Now add more information. How did he think a partnership with Spain would help him achieve this goal?,TRUE, +he wanted to defeat France and end slavey on Saint-Domingue.,"You're right that Louverture wanted to defeat France and end slavery. To make your answer better, explain how he thought an alliance with Spain would help him achieve those goals. What did Spain offer that would make it possible?",FALSE,tone +he wanted to end the fires and slavery and needed help to gain independence from France.,It's true that Louverture wanted to end slavery and gain independence from France. Now add more information. What did Spain offer to help Louverture and his troops?,TRUE, +he desired to stop the infernos and servitude and required assistance to achieve autonomy from France.,It's true that Louverture wanted to end slavery and gain independence from France. Now add more information. What did Spain offer to help Louverture and his troops?,TRUE, +he aimed to put an end to the blazes and bondage and required assistance in achieving autonomy from France.,It's true that Louverture wanted to end slavery and gain independence from France. Now add more information. What did Spain offer to help Louverture and his troops achieve these goals?,TRUE, +he desired to put an end to the flames and bondage and required assistance to achieve sovereignty from France.,It's true that Louverture wanted to end slavery and gain independence from France. Now add more information. What did Spain offer to help Louverture achieve these goals?,TRUE, +"he desired to extinguish the flames and abolish servitude, and required assistance in achieving autonomy from France.",It's true that Louverture wanted to end slavery and gain independence from France. Now add more information. What did Spain offer to help Louverture achieve these goals?,TRUE, +he desired to acquire sufficient strength to conquer the French.,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he wanted to end he fires and slavery and needed help to gain independence from France.,It's true that Louverture wanted to end slavery and gain independence from France. Now add more information. What did Spain offer to help Louverture and his troops?,TRUE, +he was not sure he could beat Fance.,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he wanted to gain enough power to defeat the Fench.,You have the right idea! Now be more specific. What did Spain offer Louverture to help him defeat France?,TRUE, +he wanted to gain enough power to defeat the French.,You have the right idea! Now be more specific. What did Spain offer Louverture to help him defeat France?,TRUE, +he wanted too end the fires and slavery and needed help too gain independence from France.,It's true that Louverture wanted to end slavery and gain independence from France. Now add more information. What did Spain offer to help Louverture and his troops?,TRUE, +he wanted to gain enought power to defeat the French.,You have the right idea! Now be more specific. What did Spain offer Louverture to help him defeat France?,TRUE, +he was not sure he could beat France.,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +his army and the Spanish shared a similar goal against the French presence in Hispaniola.,That's true! Now be more specific. What did Spain offer Louverture to help him defeat France?,TRUE, +he was uncertain he could defeat France.,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he wanted to end the fires and slaver and needed help to gain independence from France.,It's true that Louverture wanted to end slavery and gain independence from France. Now add more information. What did Spain offer to help Louverture and his troops?,TRUE, +he wanted to end the fires adn slavery adn needed help to gain independence from France.,It's true that Louverture wanted to end slavery and gain independence from France. Now add more information. What did Spain offer to help Louverture and his troops?,TRUE, +he wanted to gain enough power to defeat he French.,You have the right idea! Now be more specific. What did Spain offer Louverture to help him defeat France?,TRUE, +he wanted to end the fires and slavery and needed help to gain independence form France.,It's true that Louverture wanted to end slavery and gain independence from France. Now add more information. What did Spain offer to help Louverture and his troops?,TRUE, +He was fighting for end of slavery and needed military help.,"It's true that Louverture was fighting to end slavery. Now add more information. What kind of military help did Spain offer, and how would that help Louverture achieve his goal?",FALSE,two_directions +he wanted to unite slaves.,It's true that Louverture wanted to unite enslaved people. Now add more information. How did he think a partnership with Spain would help him achieve this goal?,FALSE,unhelpful +he was uncertain if he could defeat France.,You have the right idea that Louverture was uncertain about defeating France. Now add more information. What did Spain offer to help him defeat France?,TRUE, +he wanted to end the fires and slavery and needed help to gain independence from Fance.,It's true that Louverture wanted to end slavery and gain independence from France. Now add more information. What did Spain offer to help Louverture and his troops?,TRUE, +his army and the Spanish shared a similar goal against the Frenh presence in Hispaniola.,That's true! Now be more specific. What did Spain offer Louverture to help him defeat France?,TRUE, +he desired to acquire sufficient authority to conquer the French.,"That's true, Louverture wanted more authority. Now be more specific. How did he think a partnership with Spain would help him gain more authority and defeat the French?",FALSE,unhelpful +He was fighting for end of slavey and needed military help.,"It's true that Louverture was fighting to end slavery. Now add more information. What kind of military help did Spain offer, and how would that help Louverture achieve his goal?",FALSE,two_directions +his army and the Spanish shared a similar goal against the French presence in Hispaiola.,That's true! Now be more specific. What did Spain offer Louverture to help him defeat France?,TRUE, +he was fighting for the day when the fires could would be extinguished.,"That's true, but it doesn't explain why he formed an alliance with Spain. Why did Louverture think Spain could help him achieve his goal of ending the fighting?",TRUE, +he wanted too gain enough power too defeat the French.,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he was fighting for the day when the fires could would be extingushed.,"That's true, but it doesn't explain why Louverture formed an alliance with Spain. Reread the text and focus on what Spain offered Louverture to help him in his fight against France.",TRUE, +his army and the Spanis shared a similar goal against the French presence in Hispaniola.,That's true! Now be more specific. What did Spain offer Louverture to help him defeat France?,TRUE, +it was an opportunit to equip his soldiers.,You have the right idea! Now add another detail. What would these supplies help Louverture do?,TRUE, +his revolutionaries were under-equipped and would not be able to defeat France on their own.,You're right that Louverture's army was under-equipped. Now be more specific. What did Spain offer to help them fight France?,TRUE, +his revolutionaries were under-equipped and would not be able to defeat France on ther own.,You're right that Louverture's army was under-equipped. Now be more specific. What did Spain offer to help them fight France?,TRUE, +"it would help win cotrol of cities, ports, and land.","That was a result of Louverture's partnership with Spain. Clear your response and start again. This time, write about a reason why Louverture wanted to work with Spain.",TRUE, +"it would help win control of cities, ports, and land.","That was a result of Louverture's partnership with Spain. Clear your response and start again. This time, write about a reason why Louverture wanted to work with Spain.",TRUE, +it was an opportunity to equip his soldiers.,You have the right idea! Now add another detail. What would these supplies help Louverture do?,TRUE, +it seemed like a quicker path to freedom for his peope.,It's true that Louverture wanted freedom for his people. Now be more specific. What did Spain offer to help Louverture and his troops achieve this goal?,TRUE, +his revolutionaries were under-equipped and would not be able to defeat Frace on their own.,You're right that Louverture's army was under-equipped. Now be more specific. What did Spain offer to help them fight France?,TRUE, +his revlutionaries were under-equipped and would not be able to defeat France on their own.,You're right that Louverture's revolutionaries were under-equipped. Now be more specific. What did Spain offer to help them fight France?,TRUE, +his rebels were poorly equipped and would not have the capacity to conquer France independently.,You're right that Louverture's army was poorly equipped. Now be more specific. What did Spain offer to help Louverture and his troops?,TRUE, +it seemed like a quicker path to freedom for his people.,It's true that Louverture wanted freedom for his people. Now be more specific. What did Spain offer to help Louverture and his troops achieve this goal?,TRUE, +his insurgents were ill-equipped and would not have the capability to conquer France single-handedly.,"You're right that Louverture's army was not well-equipped. Now, be more specific. What did Spain offer to help Louverture and his troops fight the French?",TRUE, +it would make it easier too defeat France.,You have the right idea! Now be more specific. What did Spain offer Louverture to help him defeat France?,TRUE, +in order to defeat the French army and set his people free from slavery.,You're right that Louverture wanted to defeat the French army and free his people. Now add more information. How did he think a partnership with Spain would help him achieve these goals?,TRUE, +his rebels were ill-equipped and would not have the capability to conquer France independently.,"You're right that Louverture's army was not well-equipped. Now, be more specific. What did Spain offer to help Louverture and his troops fight the French?",TRUE, +Louverture knew he could not take France's powerful troops alone.,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +his rebels were ill-equipped and would not have the capability to overcome France single-handedly.,"You're right that Louverture's army was not well-equipped to fight France alone. Now, add more information about what Spain offered to help Louverture and his troops.",TRUE, +Louverture understood that he could not defeat France's formidable soldiers single-handedly.,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +it would simplify the task of conquering France.,"That's true, but it's not specific enough. What did Spain offer Louverture that would make conquering France easier?",FALSE,tone +his revolutionaries were under-equipped and would not be able too defeat France on their own.,"You're right that Louverture's army was under-equipped. Now, be more specific. What did Spain offer to help them fight France?",TRUE, +it would make it easier to defeat Frace.,You have the right idea! Now be more specific. What did Spain offer Louverture to help him defeat France?,TRUE, +it would make it easier to defeat France.,You have the right idea! Now be more specific. What did Spain offer Louverture to help him defeat France?,TRUE, +Louverture was aware that he could not engage France's formidable soldiers single-handedly.,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +it would facilitate the conquest of France.,That's true! Now be more specific. What did Spain offer Louverture to help him defeat France?,FALSE,unhelpful +his revolutionaries were under-equipped adn would not be able to defeat France on their own.,"You're right that Louverture's army was under-equipped. Now, explain how Spain's offer helped address this problem and what Louverture hoped to achieve with Spain's help.",FALSE,two_directions +Louverture was aware he could not face France's formidable soldiers single-handedly.,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +in order to defeat the French army and set his people free from slavey.,"You mentioned important goals of Louverture, but you did not explain how Spain could help him achieve them. What did Spain offer that would help Louverture defeat the French army and free enslaved people?",FALSE,tone +it would facilitate conquering France.,It's true that Louverture wanted to defeat France. Now be more specific. What did Spain offer to help Louverture defeat France?,FALSE,affirms_incorrect +in order to defeat the Frnch army and set his people free from slavery.,You're right that Louverture wanted to defeat the French army and free his people. Now add more information. How did he think a partnership with Spain would help him achieve these goals?,TRUE, +Louverture understood he could not face France's formidable soldiers single-handedly.,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +his revolutionaries where under-equipped and would not be able to defeat France on their own.,"You're right that Louverture's army was under-equipped. Now, explain how Spain's offer helped address this problem and what Louverture hoped to achieve with Spain's help.",FALSE,two_directions +in order to defeat the French army and set his peple free from slavery.,"You mentioned important goals of Louverture, but you did not explain how Spain could help him achieve them. What did Spain offer that would help Louverture defeat the French army and free enslaved people?",FALSE,tone +Louvrture knew he could not take France's powerful troops alone.,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +Louverture knew he could not take Fance's powerful troops alone.,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +Spain could help him fight France.,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +Spain could help him fight Franc.,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +Spin could help him fight France.,That's true! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +Spain could help him fght France.,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +"Spain had a common enemy in France, against whom Louverture was trying to win his peope's freedom.","That's true, but it doesn't fully explain why Louverture formed an alliance with Spain. What did Spain offer Louverture that would help him in his fight against France?",TRUE, +"Span had a common enemy in France, against whom Louverture was trying to win his people's freedom.",That's true! Now be more specific. How could Spain help Louverture defeat France?,TRUE, +"Spain had a common enemy in Frnce, against whom Louverture was trying to win his people's freedom.","That's true, but it doesn't explain why Louverture wanted to partner with Spain. What did Spain offer Louverture that would help him in his fight against France?",TRUE, +"Spain had a common enemy in France, against whom Louverture was trying to win his people's freedom.","That's true, but it doesn't fully explain why Louverture formed an alliance with Spain. What did Spain offer Louverture that would help him in his fight against France?",TRUE, +"Spain had a common enemy in France, against whom Louverure was trying to win his people's freedom.","That's true, but it doesn't fully explain why Louverture formed an alliance with Spain. The text mentions what Spain offered Louverture in exchange for his alliance. Can you add details about that offer?",FALSE,tone +Spain offered a deal that they would offer freedom.,It's true that Spain promised freedom to enslaved people. Now be more specific. What did Spanish leaders require enslaved people to do in exchange for ending slavery?,TRUE, +Spin offered a deal that they would offer freedom.,It's true that Spain promised freedom to enslaved people who joined their army. Now be more specific. What did Spanish leaders offer besides freedom?,FALSE,unhelpful +they agreed to help him defeat the French.,That's true! Now be more specific. What did Spain offer Louverture to help him defeat France?,TRUE, +they both had a common aim of diminishing French influence.,That's true! Now be more specific. How could Spain help Louverture defeat France?,TRUE, +they consented to assist him in vanquishing the French.,That's true! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +they both shared a goal of weakening French power.,That's true! Now be more specific. How could Spain help Louverture defeat France?,TRUE, +they both had an objective of diminishing French influence.,That's true! Now be more specific. How could Spain help Louverture defeat France?,TRUE, +they agreed too help him defeat the French.,That's true! Now be more specific. What did Spain offer Louverture to help him defeat France?,TRUE, +they both had a common aim to diminish the influence of France.,That's true! Now be more specific. How could Spain help Louverture defeat France?,TRUE, +they both shared a goal of weakening Fench power.,That's true! Now be more specific. How could Spain help Louverture defeat France?,TRUE, +they consented to assist him in conquering the French.,That's true! Now be more specific. What did Spain offer Louverture to help him defeat France?,TRUE, +they both had a mutual objective of diminishing French influence.,That's true! Now be more specific. How could Spain help Louverture defeat France?,TRUE, +tey both shared a goal of weakening French power.,That's true! Now be more specific. How could Spain help Louverture defeat France?,TRUE, +they agreed to help him defeat the Frech.,That's true! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +they consented to aid him in conquering the French.,That's true! Now be more specific. What did Spain offer Louverture to help him defeat France?,TRUE, +tey agreed to help him defeat the French.,"You're right that Spain agreed to help Louverture defeat the French. To make your answer better, explain what resources or support Spain offered to help him in this fight.",FALSE,tone +they both wanted to reduce the amount of power France had in Hispaniola.,That's true! Now be more specific. How could Spain help Louverture defeat France?,TRUE, +they agreed to help him defeat he French.,"You're right that Spain agreed to help Louverture defeat the French. To make your answer better, explain what resources or support Spain offered to help him in this fight.",FALSE,tone +they were both determined to diminish the level of authority France held in Hispaniola.,That's true! Now be more specific. How could Spain help Louverture defeat France?,TRUE, +they both aimed to diminish the level of influence France possessed in Hispaniola.,That's true! Now be more specific. How could Spain help Louverture defeat France?,TRUE, +they were both eager to diminish France's influence in Hispaniola.,That's true! Now be more specific. How could Spain help Louverture defeat France?,TRUE, +they were both looking to diminish the level of influence France held in Hispaniola.,That's true! Now be more specific. How could Spain help Louverture defeat France?,TRUE, +"they had a common enemy, France.",That's true! Now be more specific. How could Spain help Louverture defeat France?,TRUE, +tey both wanted to reduce the amount of power France had in Hispaniola.,That's true! Now be more specific. How could Spain help Louverture defeat France?,TRUE, +they both wanted too reduce the amount of power France had in Hispaniola.,That's true! Now be more specific. How could Spain help Louverture defeat France?,TRUE, +tey could work together to weaken the French military in Hispaniola.,That's true! Now be more specific. What did Spain offer Louverture to help him defeat France?,TRUE, +they could work together to weaken the French military in Hispaniola.,That's true! Now be more specific. What did Spain offer Louverture to help him defeat France?,TRUE, +they could work together to weaken the French military in Hispaniol.,That's true! Now be more specific. What did Spain offer Louverture to help him defeat France?,TRUE, +they could work together to weaken he French military in Hispaniola.,That's true! Now be more specific. What did Spain offer Louverture to help him defeat France?,TRUE, +they could work together to weaken the Frenh military in Hispaniola.,That's true! Now be more specific. What did Spain offer Louverture to help him defeat France?,TRUE, +they could collaborate to diminish the strength of the French army in Hispaniola.,That's true! Now be more specific. What did Spain offer Louverture to help him defeat France?,TRUE, +they needed each other's help to defeat France and regain control of Hispaniola.,That's true! Now be more specific. What did Spain offer Louverture to help him defeat France?,TRUE, +they needed each other's help to defeat France and regain control of Hispaiola.,That's true! Now be more specific. What did Spain offer Louverture to help him defeat France?,TRUE, +"they had a common enemy, Fance.",That's true! Now be more specific. How could Spain help Louverture defeat France?,TRUE, +they both wanted to reduce the amount of power France had in Hispniola.,That's true! Now be more specific. How could Spain help Louverture defeat France?,TRUE, +they both wanted to reduce the amount of power Frace had in Hispaniola.,That's true! Now be more specific. How could Spain help Louverture defeat France?,TRUE, +they could work together too weaken the French military in Hispaniola.,That's true! Now be more specific. What did Spain offer Louverture to help him defeat France?,TRUE, +they needed each other's help to defeat France and regain contrl of Hispaniola.,That's true! Now be more specific. What did Spain offer Louverture to help him defeat France?,TRUE, +they needed more fighters.,"That's true, but it's not the only reason. What did Spain offer Louverture and his army that would help them fight the French?",FALSE,affirms_incorrect +they needed each other's help to defeat Frane and regain control of Hispaniola.,That's true! Now be more specific. What did Spain offer Louverture to help him defeat France?,TRUE, +they required assistance.,You're right that Louverture needed help. Can you be more specific about what kind of assistance he needed from Spain and how that would help him in his fight against France?,FALSE,two_directions +they could collaborate to diminish the strength of the French armed forces in Hispaniola.,That's true! Now be more specific. What did Spain offer Louverture to help him defeat France?,TRUE, +they both wanted to reduce he amount of power France had in Hispaniola.,That's true! Now be more specific. How could Spain help Louverture defeat France?,TRUE, +they needed aid.,"You're right that Louverture needed help, but you need to be more specific. What kind of aid did Spain offer, and how would that aid help Louverture in his fight against France?",FALSE,two_directions +tey needed aid.,"You're right that Louverture needed help. Now be more specific. What kind of aid did Spain offer, and how would that aid help Louverture in his fight?",FALSE,two_directions +they were both against the French.,That's true! Now be more specific. How could Spain help Louverture defeat France?,TRUE, +they wanted protection.,That's true! Now be more specific. What kind of protection did Spain offer Louverture and his troops?,TRUE, +they were both against the Fench.,That's true! Now be more specific. How could Spain help Louverture defeat France?,TRUE, +to boost his authority and expel the French.,"That was a result of Louverture's partnership with Spain. Clear your response and start again. This time, write about a reason why Louverture wanted to work with Spain.",TRUE, +too increase his power and too drive out the French.,That's true! Now be more specific. What did Spain offer Louverture to help him defeat France?,TRUE, +he needed help to defeat France,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +Spain promised to end savery in Haiti,It's true that Spain promised to end slavery for some people. Now be more specific. What did Spanish leaders require enslaved people to do in exchange for ending slavery?,TRUE, +he required assistance to conquer France.,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +to increase his power and to drive out the French.,That's true! Now be more specific. What did Spain offer Louverture to help him defeat France?,TRUE, +to increase his power and to drive out the Frech.,That's true! Now be more specific. What did Spain offer Louverture to help him defeat France?,TRUE, +to increase his power adn to drive out the French.,That's true! Now be more specific. What did Spain offer Louverture to help him defeat France?,TRUE, +to increase his power and to drive out he French.,That's true! Now be more specific. What did Spain offer Louverture to help him defeat France?,TRUE, +to enhance his authority and expel the French.,"That was a result of Louverture's partnership with Spain. Clear your response and start again. This time, write about a reason why Louverture wanted to work with Spain.",TRUE, +wealth was very sought.,It's true that Saint-Domingue was a wealthy colony. Now be more specific. What did Spain offer Louverture that would help him fight France?,FALSE,unhelpful +Spain promised to end slavery in Haiti,It's true that Spain promised freedom to enslaved people who joined their army. Now be more specific. What did Spanish leaders offer that would help Louverture defeat the French?,FALSE,unhelpful +he needed help to defeat Fance,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he needed help too defeat France,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he required assistance to conquer France,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +Spin promised to end slavery in Haiti,It's true that Spain promised to end slavery for some people. Now be more specific. What did Spanish leaders require enslaved people to do in exchange for ending slavery?,TRUE, +to enhance his authority and to expel the French.,"That's true that Louverture gained authority and expelled the French from some areas. However, this is a result of his alliance with Spain, not a reason for it. Try to explain why Louverture chose to ally with Spain in the first place. What did Spain offer him that would help him achieve his goals?",TRUE, +to bolster his influence and expel the French.,"That's true that Louverture gained influence and expelled the French, but this is not why he allied with Spain. The text explains why Louverture chose to partner with Spain. Try to use information from the text to explain why Louverture formed an alliance with Spain.",FALSE,verbose +he believed it would help him defeat France,You have the right idea! Now be more specific. What did Spain offer to help Louverture defeat France?,TRUE, +he was certain that it would aid him in conquering France.,You're right that Louverture wanted to conquer France. Now be more specific. What did Spain offer to help him achieve this goal?,FALSE,affirms_incorrect +he was convinced that it would assist him in overcoming France,You have the right idea! Now be more specific. What did Spain offer to help Louverture defeat France?,TRUE, +he was convinced it would assist him in conquering France.,You have the right idea! Now be more specific. What did Spain offer to help Louverture defeat France?,FALSE,affirms_incorrect +he believed that he needed Spain's support to defeat Franc,You have the right idea! Now be more specific. What kind of support did Spain offer Louverture?,TRUE, +he believed that he needed Spain's support to defeat France,You have the right idea! Now be more specific. What kind of support did Spain offer Louverture?,TRUE, +he needed assistance too defeat France and end slavery,You have the right idea! Now be more specific. What kind of assistance did Spain offer Louverture?,TRUE, +he believed that he needed Spin's support to defeat France,You have the right idea! Now be more specific. What kind of support did Spain offer Louverture?,TRUE, +he needed assistance to defeat Frnce and end slavery,You have the right idea! Now be more specific. What kind of assistance did Spain offer Louverture?,TRUE, +he needed assistance to defeat France and end slavery,You have the right idea! Now be more specific. What kind of assistance did Spain offer to Louverture?,TRUE, +he required aid to triumph over France and terminate slavery.,You have the right idea! Now be more specific. What kind of aid did Spain offer to Louverture?,TRUE, +he was willing to do whatever it took to win freedom,It's true that Louverture was committed to ending slavery. Now add more information. How did he think a partnership with Spain would help him achieve this goal?,TRUE, +he required support to conquer France and bring an end to slavery.,You're right that Louverture wanted to end slavery and defeat France. Now add more information. What did Spain offer to help him achieve these goals?,TRUE, +he needed assistanc to defeat France and end slavery,You have the right idea! Now be more specific. What kind of assistance did Spain offer to Louverture?,TRUE, +he required help to vanquish France and terminate enslavement.,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he required help to conquer France and terminate slavery.,You're right that Louverture wanted to defeat France and end slavery. Now add more information. What did Spain offer to help him achieve these goals?,FALSE,affirms_incorrect +he needed assistance to defeat France adn end slavery,You have the right idea! Now be more specific. What kind of assistance did Spain offer to Louverture?,TRUE, +he believed it would help him defeat Franc and win abolition,It's true that Louverture wanted to defeat France and end slavery. Now add more information. What did Spain offer to help Louverture achieve these goals?,TRUE, +he needed assistance to defeat France and end slaver,You have the right idea! Now be more specific. What kind of assistance did Spain offer Louverture?,TRUE, +he believed it would help him defeat Fance,You have the right idea! Now be more specific. What did Spain offer to help Louverture defeat France?,TRUE, +he wanted assistance to conquer France.,You have the right idea! Now be more specific. What kind of assistance did Spain offer to Louverture?,FALSE,affirms_incorrect +he needed their support in the struggle against France,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he needed the help to defeat Frane,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he needed the help to defeat France,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he required their backing in the conflict versus France.,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he needed he help to defeat France,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he needed the help too defeat France,You have the right idea! Now be more specific. What kind of support did Spain offer Louverture?,TRUE, +he needed aid from a stronger force too defeat France,You have the right idea! Now be more specific. What kind of aid did Spain offer to Louverture?,TRUE, +he wanted the assistance to overcome France,You have the right idea! Now be more specific. What kind of assistance did Spain offer to Louverture?,TRUE, +he required the assistance to overpower France.,You have the right idea! Now be more specific. What kind of assistance did Spain offer to Louverture?,TRUE, +he needed ther support in the struggle against France,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he required their backing in the battle against France.,You have the right idea! Now be more specific. What kind of support did Spain offer Louverture?,TRUE, +he required assistance to conquer France.,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,FALSE,affirms_incorrect +he required assistance from a more powerful entity to conquer France.,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he needed their support in he struggle against France,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he needed aid from a stronger force to defeat France,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he needed their support in the struggle against Frace,You have the right idea! Now be more specific. What kind of support did Spain offer Louverture?,TRUE, +he needed their suport in the struggle against France,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he required assistance from a more powerful entity to overcome France.,You have the right idea! Now be more specific. What kind of assistance did Spain offer to Louverture?,TRUE, +he required their assistance in vanquishing the French.,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he required their backing during the conflict with France.,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he required their assistance to vanquish France.,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he needed aid form a stronger force to defeat France,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he required help from a more powerful entity to overcome France.,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he needed their help to defeat France,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he needed ther help to defeat France,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he required their assistance to conquer France.,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,FALSE,affirms_incorrect +he needed aid from a stronger force to defeat Frane,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he needed their help too defeat France,You have the right idea! Now be more specific. What kind of support did Spain offer Louverture?,TRUE, +he needed their help to defeat Fance,You have the right idea! Now be more specific. What kind of support did Spain offer Louverture?,TRUE, +he required comrades to aid in conquering France,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,FALSE,affirms_incorrect +he needed allies to help defeat France,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he needed allies to help defeat Franc,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he required comrades to aid in conquering France.,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,FALSE,affirms_incorrect +he required comrades to assist in overcoming France.,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he required supporters to assist in conquering France,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,FALSE,affirms_incorrect +he knew that he couldn't defeat Frnce alone,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he believed it could help him defeat Franc and end slavery in Saint-Domingue,It's true that Louverture wanted to defeat France and end slavery. Now add more information. How did he think a partnership with Spain would help him achieve these goals?,TRUE, +he believed it was the quickest path to victory against France,It's true that Louverture wanted to defeat France. Now be more specific. What did Spain offer to help him achieve this goal?,TRUE, +he knew that he couldn't defeat France alone,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he needed allies too help defeat France,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he believed it was the quickest path to victory against Frace,It's true that Louverture wanted to defeat France. Now be more specific. What did Spain offer to help him achieve this goal?,TRUE, +Spain promised to help him end slvery in Saint-Domingue.,It's true that Spain's promise to free enslaved people who joined their army was important to Louverture. Now add more information. What other resources did Spain offer that would help Louverture defeat the French?,FALSE,unhelpful +Spai promised to help him end slavery in Saint-Domingue.,It's true that Spain's promise to free enslaved people who joined their army was important to Louverture. Now add more information. What other resources did Spain offer that would help Louverture defeat the French?,FALSE,unhelpful +Spain promised to help him end slavery in Saint-Domingue.,It's true that Spain's promise to free enslaved people who joined their army was important to Louverture. Now add more information. What resources did Spain offer that would help Louverture fight the French?,FALSE,unhelpful +he saw it as the best way to defeat France and end slavery,It's true that Louverture wanted to defeat France and end slavery. Now add more information. What did Spain offer to help Louverture achieve these goals?,TRUE, +he perceived it as the optimal method to conquer France and put an end to slavery.,That's true! Now be more specific. What did Spain offer Louverture to help him defeat France and end slavery?,TRUE, +Spain promised to help him end slavery in Sant-Domingue.,It's true that Spain's promise to free enslaved people who joined their army was important to Louverture. Now add more information. What resources did Spain offer that would help Louverture fight the French?,FALSE,gives_answer +Spain promised to help him end slavery in Saint-Domigue.,It's true that Spain's promise to free enslaved people who joined their army was important to Louverture. Now add more information. What resources did Spain offer that would help Louverture fight the French?,FALSE,gives_answer +he perceived it as the ideal method to vanquish France and put an end to slavery.,It's true that Louverture wanted to defeat France and end slavery. Now be more specific. What did Spain offer to help Louverture achieve these goals?,TRUE, +he regarded it as the most optimal strategy to vanquish France and put an end to the practice of slavery.,It's true that Louverture wanted to defeat France and end slavery. Now add more information. What did Spain offer to help Louverture achieve these goals?,TRUE, +he perceived it as the optimal solution to beat France and terminate slavery.,That's true! Now be more specific. How could Spain help Louverture defeat France and end slavery?,TRUE, +he saw it as the best way too defeat France and end slavery,It's true that Louverture wanted to defeat France and end slavery. Now add more information. How did he think a partnership with Spain would help him achieve these goals?,TRUE, +he saw it as he best way to defeat France and end slavery,It's true that Louverture wanted to defeat France and end slavery. Now add more information. What did Spain offer to help Louverture achieve these goals?,TRUE, +he saw it as the best way to defeat Frane and end slavery,It's true that Louverture wanted to defeat France and end slavery. Now add more information. What did Spain offer to help Louverture achieve these goals?,TRUE, +he saw it as the best way to defeat France adn end slavery,It's true that Louverture wanted to defeat France and end slavery. Now be more specific. How did he think a partnership with Spain would help him achieve these goals?,TRUE, +he needed support to fight France,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he thought it could help him defeat France adn end slavery in Saint-Domingue,It's true that Louverture wanted to defeat France and end slavery. Now add more information. What did Spain offer to help him achieve these goals?,TRUE, +he required assistance to combat France,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he thought it could help him defeat Fance and end slavery in Saint-Domingue,It's true that Louverture wanted to defeat France and end slavery. Now add more information. What did Spain offer to help Louverture achieve these goals?,TRUE, +he saw it as the best way to defeat France and end slavry,It's true that Louverture wanted to defeat France and end slavery. Now be more specific. How did he think a partnership with Spain would help him achieve these goals?,TRUE, +he believed it had the potential to aid him in conquering France and abolishing slavery in Saint-Domingue.,It's true that Louverture wanted to defeat France and end slavery. Now be more specific. What did Spain offer to help Louverture achieve these goals?,FALSE,affirms_incorrect +he needed support to fight Fance,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he required assistance in battling France.,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he needed support to fiht France,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he required assistance to combat France.,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he thought it could help him defeat France and end slavery in Saint-Dominge,It's true that Louverture wanted to defeat France and end slavery. Now add more information. What did Spain offer to help him achieve these goals?,TRUE, +he needed support too fight France,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he needed suport to fight France,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he needed more weapons and food,You have the right idea! Now add another detail. What could the weapons and food help Louverture do?,TRUE, +Spai could provide much-needed resources for the revolutionaries,You have the right idea! Now add another detail. What would these resources help Louverture do?,TRUE, +freedom was the ultimate goal,It's true that Louverture wanted freedom for the enslaved people. Now add more information. How did he think a partnership with Spain would help him achieve this goal?,TRUE, +Spain could provide much-needed resources for the revolutionaries,"You're right that Spain could provide resources. Now be more specific. What kind of resources did Spain offer, and how would those resources help Louverture fight the French?",FALSE,two_directions +Spain could provide much-needed resources for the revolutonaries,"You're right that Spain could provide resources. Now be more specific. What kinds of resources did Spain offer, and how would those resources help Louverture fight the French?",FALSE,two_directions +Spain had the resources he needed.,"You have the right idea! Now add another detail. What resources did Spain have, and what would these resources help Louverture do?",FALSE,two_directions +Span had the resources he needed.,"You have the right idea! Now add another detail. What resources did Spain have, and what would these resources help Louverture do?",FALSE,two_directions +he was aware that he required aid in order to overcome France alongside his troops.,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he wanted to end slavery in the French colony of Saint-Domingue before Frnce would,It's true that Louverture wanted to end slavery. Now add more information. How did he think a partnership with Spain would help him achieve this goal?,TRUE, +he knew he needed assistance if he adn his army were to defeat France,You have the right idea! Now be more specific. What kind of assistance did Spain offer to Louverture?,TRUE, +he understood he required help if he and his troops were to conquer France,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,FALSE,affirms_incorrect +he needed support in his figt against France,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he knew he needed assistance if he and his army were too defeat France,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he wanted to end slavery in the French colny of Saint-Domingue before France would,It's true that Louverture wanted to end slavery. Now add more information. How did he think a partnership with Spain would help him achieve this goal?,TRUE, +he wanted to end slavery in the Fench colony of Saint-Domingue before France would,It's true that Louverture wanted to end slavery. Now add more information. How did he think a partnership with Spain would help him achieve this goal?,TRUE, +he knew he needed assistance if he and his army where to defeat France,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he realized he required aid to overpower France with his troops.,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he wanted to end slavery in the French colony of Saint-Dominge before France would,It's true that Louverture wanted to end slavery. Now add more information. How did he think a partnership with Spain would help him achieve this goal?,TRUE, +he knew he needed assistance if he and his army were to defeat Frnce,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he needed support in his fight against France,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he was aware that he required aid in order for him and his troops to overcome France.,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he wanted to end slaery in the French colony of Saint-Domingue before France would,It's true that Louverture wanted to end slavery. Now add more information. How did he think a partnership with Spain would help him achieve this goal?,TRUE, +he wanted to end slavery in the French colony of Saint-Domingue before France would,It's true that Louverture wanted to end slavery. Now add more information. How did he think a partnership with Spain would help him achieve this goal?,TRUE, +he knew he needed assistance if he and his army were to defeat France,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he knew he needed assistane if he and his army were to defeat France,You have the right idea! Now be more specific. What kind of assistance did Spain offer to Louverture?,TRUE, +he needed support in his fight against Fance,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he wanted to end slavery in the French colony of Sint-Domingue before France would,It's true that Louverture wanted to end slavery. Now add more information. How did he think a partnership with Spain would help him achieve this goal?,TRUE, +they both wanted to get rid of French control over Hispaniola,That's true! Now be more specific. How could Spain help Louverture defeat France?,TRUE, +he believed it would help him defeat Franc and the slave system,It's true that Louverture wanted to defeat France and end slavery. Now add more information. What did Spain offer to help Louverture achieve these goals?,TRUE, +they were both eager to eliminate French domination from Hispaniola.,That's true! Now be more specific. How could Spain help Louverture defeat France?,TRUE, +tey both wanted to get rid of French control over Hispaniola,That's true! Now be more specific. How could Spain help Louverture defeat France?,TRUE, +they both desired to eliminate the French domination of Hispaniola.,That's true! Now be more specific. How could Spain help Louverture defeat France?,TRUE, +they were both eager to eliminate French dominance over Hispaniola.,That's true! Now be more specific. How could Spain help Louverture defeat France?,TRUE, +they both wanted to get rid of French conrol over Hispaniola,That's true! Now be more specific. How could Spain help Louverture defeat France?,TRUE, +they both wanted to get rid of French control over Hispanila,That's true! Now be more specific. How could Spain help Louverture defeat France?,TRUE, +they were both eager to eliminate French dominance on Hispaniola,That's true! Now be more specific. How could Spain help Louverture defeat France?,TRUE, +they both wanted to get rid of Frnch control over Hispaniola,That's true! Now be more specific. How could Spain help Louverture defeat France?,TRUE, +they both wanted too get rid of French control over Hispaniola,That's true! Now be more specific. How could Spain help Louverture defeat France?,TRUE, +he had too defeat France too end slavery,You have the right idea! Now be more specific. What did Spain offer Louverture to help him defeat France?,TRUE, +he had to defeat France to end slavery,"You're right that Louverture wanted to end slavery, but this response doesn't explain why he formed an alliance with Spain. How did he think an alliance with Spain would help him defeat France and end slavery?",TRUE, +he was required to conquer France in order to put an end to the practice of enslavement.,"That's true, Louverture wanted to end slavery. But this response doesn't explain why he formed an alliance with Spain. How did he think Spain could help him achieve his goal of ending slavery?",FALSE,affirms_incorrect +he had to defeat France to end slvery,"You're right that Louverture wanted to end slavery, but this response doesn't explain why he formed an alliance with Spain. How did he think an alliance with Spain would help him defeat France and end slavery?",TRUE, +he had to conquer France in order to abolish slavery.,"That's true, but it doesn't explain why he formed an alliance with Spain. How did Spain help him in his fight against France?",FALSE,affirms_incorrect +he had to conquer France to put an end to slavery,That's true! Now be more specific. How could Spain help Louverture defeat France?,FALSE,affirms_incorrect +he had to defeat Fance to end slavery,"You're right that Louverture wanted to end slavery, but this response doesn't explain why he formed an alliance with Spain. What did Spain offer that would help him defeat France and achieve his goal?",TRUE, +he needed support for his army,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he needed to conquer France in order to put an end to slavery,"You're right that Louverture wanted to end slavery, and defeating France was part of that plan. But your answer doesn't explain why he chose to ally *with Spain* to do that. What did Spain offer that would help him fight France?",TRUE, +he required assistance for his military forces,"You have the right idea! Now be more specific. What kind of assistance did Louverture need, and how did Spain offer to provide it?",FALSE,two_directions +he needs help to defeat the French army and win control of the Caribbean island of Hispaniola,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he needed suport for his army,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he required backing for his military force.,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he required backing for his military forces.,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he required aid for his military,"You have the right idea! Now be more specific. What kind of aid did Louverture need, and how did Spain offer to provide it?",FALSE,two_directions +he needs help to defeat the French army and win cotrol of the Caribbean island of Hispaniola,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he needs help to defeat the French army and win control of the Caribbean islad of Hispaniola,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he needs help to defeat the Frenh army and win control of the Caribbean island of Hispaniola,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he needs help to defeat the French army and win control of the Caribbean island of Hispniola,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +Spain and France had become enemies,That's true! Now be more specific. How could Spain help Louverture defeat France?,TRUE, +he was hoping that Spain would assist him in defeating France,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +Spain and Frane had become enemies,That's true! Now be more specific. How could Spain help Louverture defeat France?,TRUE, +Spai and France had become enemies,That's true! Now be more specific. How could Spain help Louverture defeat France?,TRUE, +he was hoping that Spin would assist him in defeating France,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +Spain pledged to assist him and his comrades in their efforts to conquer France during their conflict on Saint-Domingue.,That's true! Now be more specific. What kind of support did Spain offer to Louverture?,FALSE,affirms_incorrect +he was hoping that Spain would assist him in defeating Franc,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +it would help him in his quest to defeat France and free the enslaved people of Saint-Domingue,It's true that Louverture wanted to defeat France and free the enslaved people. Now be more specific. What did Spain offer to help him achieve these goals?,TRUE, +it would help him in his quest to defeat France and free the enslaved people of Saint-Dominge,It's true that Louverture wanted to defeat France and free enslaved people. Now add more information. What did Spain offer to help Louverture achieve these goals?,TRUE, +it would help him in his quest to defeat France and free the enslaved poeple of Saint-Domingue,It's true that Louverture wanted to defeat France and free the enslaved people. Now add more information. What did Spain offer to help Louverture achieve these goals?,TRUE, +it would help him in his quest to defeat France and free the enslaved peope of Saint-Domingue,It's true that Louverture wanted to defeat France and free the enslaved people. Now add more information. What did Spain offer to help Louverture achieve these goals?,TRUE, +it would help him in his quest to defeat France and free the eslaved people of Saint-Domingue,It's true that Louverture wanted to defeat France and free enslaved people. Now add more information. What did Spain offer to help Louverture achieve these goals?,TRUE, +it would help him in his quest to defeat France adn free the enslaved people of Saint-Domingue,It's true that Louverture wanted to defeat France and free the enslaved people. Now be more specific. What did Spain offer to help Louverture achieve these goals?,TRUE, +Spain promised to end French control of the colony of Sant-Dominique (now Haiti) and free all the Black slaves,"Spain's goals were to weaken France and regain control of the island, but that's not why Louverture formed an alliance. What did Spain offer Louverture and his troops that would help them fight the French?",TRUE, +Spain promised to end French control of the coony of Saint-Dominique (now Haiti) and free all the Black slaves,"Spain's goals were to weaken France and regain control of the island, but that's not why Louverture formed an alliance. What did Spain offer Louverture and his troops that would help them fight the French?",TRUE, +Sain promised to end French control of the colony of Saint-Dominique (now Haiti) and free all the Black slaves,"That's true that Spain and Louverture shared a common enemy, France. However, your response doesn't explain why Louverture chose to ally with Spain. What did Spain offer Louverture that made the alliance beneficial to him?",TRUE, +Spain was providing aid in bringing down France,That's true! Now be more specific. What kind of aid did Spain offer Louverture?,TRUE, +Spain was offering help to take down France,That's true! Now be more specific. What kind of help did Spain offer Louverture?,TRUE, +Spain promised to end French contro of the colony of Saint-Dominique (now Haiti) and free all the Black slaves,"Spain's goals were to weaken France and regain control of the island, but that's not why Louverture formed an alliance. What did Spain offer Louverture and his troops that would help them fight the French?",TRUE, +Spain promised to end Fench control of the colony of Saint-Dominique (now Haiti) and free all the Black slaves,"Spain's goals were to weaken France and regain control of the island. Your answer mentions Spain's goals, but not Louverture's reasons for allying with them. Reread the text to find what Spain offered Louverture to help him fight the French.",FALSE,tone +Spain was extending aid to bring down France.,That's true! Now be more specific. What kind of aid did Spain offer Louverture?,TRUE, +Spain promised to end French control of the colony of Saint-Dominique (now Haiti) and free all the Black slaves,"That's true that Spain and Louverture shared a common enemy, France. However, the text mentions other reasons why Louverture formed an alliance with Spain. Can you find those reasons in the text and use them to answer the question?",FALSE,tone +Spain was proposing aid to bring down France,That's true! Now be more specific. What kind of aid did Spain offer Louverture?,TRUE, +Spain was offering help too take down France,That's true! Now be more specific. What kind of help did Spain offer Louverture?,TRUE, +it would help him in his quest to defeat Frace and free the enslaved people of Saint-Domingue,It's true that Louverture wanted to defeat France and free enslaved people. Now add more information. What did Spain offer to help Louverture achieve these goals?,TRUE, +Spain was presenting assistance to bring down France.,That's true! Now be more specific. What kind of assistance did Spain offer Louverture?,TRUE, +Spin was offering help to take down France,That's true! Now be more specific. What kind of help did Spain offer Louverture?,TRUE, +Spain was offering help to take down Frace,That's true! Now be more specific. What kind of help did Spain offer Louverture?,TRUE, +he wanted to defeat the French who had a strong army.,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he believed it could assist him in emerging victorious in battles against France.,You're right that Louverture wanted to defeat France. Now be more specific. What did Spain offer to help him achieve this goal?,TRUE, +he wanted to defeat the Frenc who had a strong army.,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +Spanish officials offered to help and support in fighting for the freedom of the people of Saint-Domingue,That's true! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +Authorities from Spain proposed their assistance and backing in the struggle for the liberty of the inhabitants of Saint-Domingue.,That's true! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +"he aimed to conquer the French, whose military was formidable.","You're right that Louverture wanted to defeat the French, but you need to explain how Spain could help him do that. What did Spain offer to help Louverture and his troops fight the French?",FALSE,tone +he wanted to defeat he French who had a strong army.,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +Spanish authorities proposed to assist and endorse the struggle for the liberation of the inhabitants of Saint-Domingue.,"That's true, but it doesn't explain why Louverture formed an alliance with Spain. What did Spain offer that would help Louverture in his fight against France?",TRUE, +he wanted too defeat the French who had a strong army.,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +Spain was a main rival of France and an ally of Great Britain,"That's true, but it doesn't explain why Louverture formed an alliance with Spain. The text explains what Spain offered Louverture to help him fight the French. Try again, focusing on what Spain offered Louverture.",FALSE,verbose +Spanish officials offered to help and support in fighting for he freedom of he people of Saint-Domingue,That's true! Spain did want to help Louverture. Now be more specific. What did Spain offer to help Louverture and his troops fight the French?,TRUE, +Spin was a main rival of France and an ally of Great Britain,"That's true, but it doesn't explain why Louverture formed an alliance with Spain. Reread the text and focus on what Spain offered Louverture to help him fight the French.",TRUE, +Spanish officials offered too help and support in fighting for the freedom of the people of Saint-Domingue,That's true! Now be more specific. What kind of help and support did Spain offer Louverture and his troops?,TRUE, +Spain was a main rival of Frace and an ally of Great Britain,"That's interesting, but the text doesn't say that Spain was allied with Great Britain. The text does mention that Spain and France were rivals. Can you use information from the text to explain why Louverture allied with Spain?",FALSE,tone +"he aimed to conquer the French, who possessed a formidable military force.","You're right that Louverture wanted to defeat the French, but you need to explain how Spain could help him do that. What did Spain offer to help Louverture and his troops fight the French?",FALSE,affirms_incorrect +Spanish officials offered to help and suport in fighting for the freedom of the people of Saint-Domingue,That's true! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +"he desired to conquer the French, who possessed a formidable military force.","You're right that Louverture wanted to defeat the French, but you need to explain how Spain could help him do that. What did Spain offer that would help Louverture and his troops fight the French?",FALSE,affirms_incorrect +the Spanish authorities extended their assistance and backing in the struggle for the liberation of the inhabitants of Saint-Domingue.,That's true! Now be more specific. What kind of assistance and backing did Spain offer Louverture and his troops?,TRUE, +Spanish officials offered to help and support in fighting for the freedom of the poeple of Saint-Domingue,That's true! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +Spanish officials offered to help adn support in fighting for the freedom of the people of Saint-Domingue,That's true! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +Spnish officials offered to help and support in fighting for the freedom of the people of Saint-Domingue,That's true! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +Spanish officials offered to help and support in fighting for the freedom of the people of Saint-Dmingue,That's true! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +Spanish officials offered to help and support in fighting for the freedom of the people of Sait-Domingue,That's true! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +Spanish officials offered to help and support in fighting for the freedom of the peole of Saint-Domingue,That's true! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he needed help defeating Frane,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he needed help defeating the French army in Saint-Domingu,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he thought it could help him win the fights against France,You have the right idea! Now be more specific. What did Spain offer to help Louverture and his troops fight France?,TRUE, +he thought it could help him win the fights against Franc,You have the right idea! Now be more specific. What did Spain offer to help Louverture and his troops fight France?,TRUE, +he believed it might assist him in emerging victorious in the battles against France.,You're right that Louverture wanted to defeat France. Now add more information. What did Spain offer that would help him achieve this goal?,TRUE, +he needed help defeating the Frenc army in Saint-Domingue,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he needed help defeating the French army in Saint-Domingue,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he needed help defeating France,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he believed it would help him win freedom from France,It's true that Louverture wanted freedom from France. Now add more information. What did Spain offer to help Louverture and his troops fight France?,TRUE, +he believed it could assist him in emerging victorious in the battles against France.,It's true that Louverture wanted to defeat France. Now be more specific. What did Spain offer to help him achieve this goal?,TRUE, +he believed it would help him win freedom from Fance,It's true that Louverture wanted freedom from France. Now add more information. What did Spain offer to help Louverture and his troops fight France?,TRUE, +he needed help defeating the French army in Sint-Domingue,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he thought it could help him win he fights against France,You have the right idea! Now be more specific. What did Spain offer to help Louverture and his troops fight the French?,TRUE, +he needed assistance in the fight against France,You have the right idea! Now be more specific. What kind of assistance did Spain offer to Louverture?,TRUE, +he needed assistance in the figt against France,You have the right idea! Now be more specific. What kind of assistance did Spain offer to Louverture?,TRUE, +he needed assistance in the fight against Frace,You have the right idea! Now be more specific. What kind of assistance did Spain offer to Louverture?,TRUE, +he couldn't defeat France on his own,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he needed assitance in the fight against France,You have the right idea! Now be more specific. What kind of assistance did Spain offer to Louverture?,TRUE, +he believed it would help him achieve abolition in Sait-Domingue,It's true that Louverture wanted to end slavery. Now add more information. How did he think a partnership with Spain would help him achieve this goal?,TRUE, +he couldn't defeat Frnce on his own,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he believed it would help him achieve abolition in Saint-Domingue,It's true that Louverture wanted to end slavery. Now add more information. How did he think a partnership with Spain would help him achieve this goal?,TRUE, +he believed it would help him achieve abolition in Saint-Dominue,It's true that Louverture wanted to end slavery. Now add more information. How did he think a partnership with Spain would help him achieve this goal?,TRUE, +he believed it was the best path to victory in the Haitian Reolution,It's true that Louverture wanted to win the revolution. Now be more specific. What did Spain offer that would help him achieve this goal?,TRUE, +he believed it was the best path to victory in the Haitian Revolution,It's true that Louverture wanted to win the revolution. Now be more specific. What did Spain offer to help him achieve this goal?,TRUE, +he needed its support to achieve his goal of freedom,You're right that Louverture wanted freedom for the enslaved people. Now add more information. What kind of support did Spain offer that would help him achieve that goal?,TRUE, +Spain would help him win the war against France,That's true! Now be more specific. What resources did Spain offer to help Louverture defeat France?,TRUE, +he wanted to decrease the number of Spanish soldiers in Sain-Domingue,That's not quite right. The text explains that Spain wanted to weaken France's power in Hispaniola. Why did Louverture think partnering with Spain would help *him* achieve his goals?,FALSE, +he wanted to decrease the number of Spanish soldiers in Saint-Domingue,That's not quite right. The text explains that Spain wanted to decrease the number of French soldiers in Saint-Domingue. Why did Louverture want to partner with Spain? Look for information in the text about what Spain offered Louverture and his troops.,FALSE, +he wanted to decrease the number of Spanish soldiers in Saint-Domngue,That's not quite right. The text explains that Spain wanted to weaken France's power in Hispaniola. Why did Louverture think partnering with Spain would help *him* achieve his goals?,FALSE, +he wanted to decrease the number of Spansh soldiers in Saint-Domingue,That's not quite right. The text explains that Spain wanted to weaken France's power in Hispaniola. Why did Louverture think partnering with Spain would help *him* achieve his goals?,FALSE, +Spain would assist him in achieving victory in the battle against France.,That's true! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +Spain would assist him in conquering the battle against France.,That's true! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +Spin would help him win the war against France,That's true! Now be more specific. What did Spain offer Louverture to help him defeat France?,TRUE, +Spain would help him win he war against France,That's true! Now be more specific. What resources did Spain offer to help Louverture defeat France?,TRUE, +Spain would help him win the war against Frace,That's true! Now be more specific. What resources did Spain offer to help Louverture defeat France?,TRUE, +Spanish leaders said they were going to end slavery in Saint-Domingue,It's true that Spain promised to end slavery for some people. Now be more specific. What did Spanish leaders require enslaved people to do in exchange for ending slavery?,TRUE, +Spanish leaders said they were going to end slavery in Sint-Domingue,It's true that Spain promised to end slavery for some people. Now be more specific. What did Spanish leaders require enslaved people to do in exchange for ending slavery?,TRUE, +Spanish leaders said they were going to end slaver in Saint-Domingue,It's true that Spain promised to end slavery for some people. Now be more specific. What did Spanish leaders require enslaved people to do in exchange for ending slavery?,TRUE, +Spanish leaders said they were going to end slavery in Saint-Dominge,It's true that Spain promised to end slavery for some people. Now be more specific. What did Spanish leaders require enslaved people to do in exchange for ending slavery?,TRUE, +Spanis leaders said they were going to end slavery in Saint-Domingue,It's true that Spain promised to end slavery for some people. Now be more specific. What did Spanish leaders require enslaved people to do in exchange for ending slavery?,TRUE, +Spain promised to help Louvrture defeat France in the Haitian Revolution,That's true! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +Spain promised to help Louverture defeat France in the Haitian Revolution,That's true! Now be more specific. What resources did Spain offer to help Louverture defeat France?,TRUE, +i don't knooooooow,It seems like you're having trouble finding the answer. Let's look at the text together. What did Spain offer Louverture in exchange for his support?,TRUE, +Spain promised to help Louverture defeat Fance in the Haitian Revolution,That's true! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +Spain promised to help Louverture defeat France in the Haitian Revlution,That's true! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +Span promised to help Louverture defeat France in the Haitian Revolution,That's true! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +"they possessed what he required to conquer France, which was some sort of support.",You're right that Spain had something Louverture needed to fight France. Can you be more specific about what kind of support Spain offered?,FALSE,tone +"they had what he needed too defeat France, which was some form of backing",You're right that Spain provided support to help defeat France. Can you be more specific about what kind of backing Spain offered? The text mentions several things.,FALSE,tone +"they had what he needed to defeat Frane, which was some form of backing",You're right that Spain provided support to help defeat France. Can you be more specific about what kind of backing Spain offered Louverture?,FALSE,tone +France was not willing to give up control of Saint-Domingue without a figt,That's true! Now be more specific. What did Spain offer Louverture to help him defeat France?,TRUE, +France was not willing to give up control of Sain-Domingue without a fight,That's true! Now be more specific. What did Spain offer Louverture to help him defeat France?,TRUE, +France was not willing to give up cntrol of Saint-Domingue without a fight,That's true! Now be more specific. What did Spain offer Louverture to help him defeat France?,TRUE, +Franc was not willing to give up control of Saint-Domingue without a fight,That's true! Now be more specific. What did Spain offer Louverture to help him defeat France?,TRUE, +"tey had what he needed to defeat France, which was some form of backing",You're right that Spain had what Louverture needed to defeat France. Can you be more specific about what resources or supplies Spain offered? How would those resources help him fight the French?,FALSE,two_directions +France was not willing to give up control of Saint-Domingue without a fight,That's true! Now be more specific. What did Spain offer Louverture to help him defeat France?,TRUE, +the Spanish were willing to help the Haitian Revolutionaries,That's true! Now be more specific. What did Spain offer to help Louverture and his troops?,TRUE, +the Spansh were willing to help the Haitian Revolutionaries,That's true! Now be more specific. What did Spain offer to help Louverture and his troops?,TRUE, +France refused to give up control of Sait-Domingue,That's true! Now be more specific. What did Spain offer Louverture to help him defeat France?,TRUE, +Frace was unwilling to give up control of Saint-Domingue without a fight,That's true! Now be more specific. What did Spain offer Louverture to help him defeat France?,TRUE, +France was unwilling to give up control of Sint-Domingue without a fight,That's true! Now be more specific. What did Spain offer Louverture to help him defeat France?,TRUE, +Spai needed additional forces in Hispaniola to defeat France.,"That's a reason why Spain wanted to form a partnership. Clear your response and start again. This time, write about a reason why Louverture wanted to build a relationship.",TRUE, +Spain needed additional forces in Hispaniola to defeat France.,"That's a reason why Spain wanted to form a partnership. Clear your response and start again. This time, write about a reason why Louverture wanted to build a relationship.",TRUE, +Spain needed additional forces in Hispanila to defeat France.,"That's a reason why Spain wanted to form a partnership. Clear your response and start again. This time, write about a reason why Louverture wanted to build a relationship.",TRUE, +he saw it as the best way to defeat France and end slavery in Saint-Domingue,It's true that Louverture wanted to defeat France and end slavery. Now add more information. How did he think a partnership with Spain would help him achieve these goals?,TRUE, +Spain would help him overthrow Frane,That's true! Now be more specific. How could Spain help Louverture defeat France?,TRUE, +Spai would help him overthrow France,That's true! Now be more specific. How could Spain help Louverture defeat France?,TRUE, +he saw it as the best way to defeat France and end slavery in Saint-Dominue,It's true that Louverture wanted to defeat France and end slavery. Now add more information. What did Spain offer to help Louverture achieve these goals?,TRUE, +he saw it as the best way to defeat Frace and end slavery in Saint-Domingue,It's true that Louverture wanted to defeat France and end slavery. Now add more information. What did Spain offer to help Louverture achieve these goals?,TRUE, +France was not willing to give up control of Saint-Dominge without a fight,That's true! Now be more specific. What did Spain offer Louverture to help him defeat France?,TRUE, +"the French were unwilling to give up conrol of Saint-Domingue, their most valuable colony",That's true! Now be more specific. What did Spain offer Louverture to help him defeat France?,TRUE, +"the French were unwilling to give up control of Saint-Domingue, their most valuable colny",That's true! Now be more specific. What did Spain offer Louverture to help him defeat France?,TRUE, +"the Frenh were unwilling to give up control of Saint-Domingue, their most valuable colony",That's true! Now be more specific. What did Spain offer Louverture to help him defeat France?,TRUE, +"the French were unwilling to give up control of Saint-Domingue, their most valuable colony",That's true! Now be more specific. What did Spain offer Louverture to help him defeat France?,TRUE, +Spai was the most powerful colonial power in the Caribbean,"That's true, but it's not the reason Louverture formed an alliance with Spain. The text explains why Louverture needed Spain's help. Can you find that information and use it to answer the question?",FALSE,tone +"the French were unwilling to give up control of Sint-Domingue, their most valuable colony",That's true! Now be more specific. What did Spain offer Louverture to help him defeat France?,TRUE, +the Frnch had been unresponsive to their demands for freedom and equality,"That's true, but this response doesn't explain why Louverture formed an alliance with Spain. The text explains what Spain offered Louverture in exchange for his alliance. Try to use that information to answer the question.",FALSE,tone +France was unwilling to give up conrol of Saint-Domingue without a fight,That's true! Now be more specific. What did Spain offer Louverture to help him defeat France?,TRUE, +Fance refused to give up control of Saint-Domingue,That's true! Now be more specific. What did Spain offer Louverture to help him defeat France?,TRUE, +France was unwilling to give up control of Saint-Domingue without a figh,That's true! Now be more specific. What did Spain offer Louverture to help him defeat France?,TRUE, +France refused to give up cntrol of Saint-Domingue,That's true! Now be more specific. What did Spain offer Louverture to help him defeat France?,TRUE, +"his troops were running low on weapons, amunition, and food",You have the right idea! Now add another detail. What could these resources help Louverture do?,TRUE, +Spain required a larger fighting force to weaken French power in Hispaniola,"That's a reason why Spain wanted to form a partnership. Clear your response and start again. This time, write about a reason why Louverture wanted to build a relationship.",TRUE, +he saw it as the best way to defeat France and end slavery in Sain-Domingue,It's true that Louverture wanted to defeat France and end slavery. Now add more information. What did Spain offer to help him achieve these goals?,TRUE, +Spain would help him overthrow France,That's true! Now be more specific. How could Spain help Louverture defeat France?,TRUE, +"his troops were running low on weapons, ammunition, and food",You have the right idea! Now add another detail. What could these resources help Louverture do?,TRUE, +Spai required a larger fighting force to weaken French power in Hispaniola,"That's a reason why Spain wanted to form a partnership. Clear your response and start again. This time, write about a reason why Louverture wanted to build a relationship.",TRUE, +he saw it as the best way to defeat France and end slvery in Saint-Domingue,It's true that Louverture wanted to defeat France and end slavery. Now add more information. What did Spain offer to help Louverture achieve these goals?,TRUE, +the French had been unresponsive to their demands for freedom and equality,"That's true, but it doesn't explain why Louverture allied with Spain. The text explains what Spain offered Louverture to help him fight the French. Try again, focusing on what Spain offered Louverture.",FALSE,verbose +France was unwilling to give up control of Saint-Dmingue without a fight,That's true! Now be more specific. What did Spain offer Louverture to help him defeat France?,TRUE, +Spain needed additional forces in Hispaniola to defeat Fance.,"That's a reason why Spain wanted to form a partnership. Clear your response and start again. This time, write about a reason why Louverture wanted to build a relationship.",TRUE, +Spain was the most powerful colonial power in the Caribbean,That's not quite right. The text states that Spain had a shared goal with Louverture: weakening French power in Hispaniola. Can you explain how Spain's resources could help Louverture achieve that goal?,TRUE, +"the French were unwilling to give up control of Saint-Dominue, their most valuable colony",That's true! Now be more specific. What did Spain offer Louverture to help him defeat France?,TRUE, +Spain required a larger fighting force to weaken Frenh power in Hispaniola,"That's a reason why Spain wanted to form a partnership. Clear your response and start again. This time, write about a reason why Louverture wanted to build a relationship.",TRUE, +Spain required a larger fighting force to weaken French power in Hipaniola,"That's a reason why Spain wanted to form a partnership. Clear your response and start again. This time, write about a reason why Louverture wanted to build a relationship.",TRUE, +he was convinced that they would back his struggle for liberty against the French settlers.,That's true! Now be more specific. What did Spain offer Louverture to help him defeat France?,TRUE, +he believed that they would support his fight for freedom against the French colonists,It's true that Spain supported Louverture's fight for freedom. Now be more specific. What did Spain offer to help Louverture and his troops?,TRUE, +he believed that tey would support his fight for freedom against the French colonists,It's true that Spain supported Louverture's fight for freedom. Now be more specific. What did Spain offer to help Louverture and his troops?,TRUE, +he believed that they would support his fight for freedom against he French colonists,It's true that Spain supported Louverture's fight for freedom. Now be more specific. What did Spain offer to help Louverture and his troops?,TRUE, +Spain was engaged in a conflict with France during that period.,That's true! Now be more specific. How could Spain help Louverture defeat France?,TRUE, +he believed that they would support his fight for freedom against the Frech colonists,It's true that Spain supported Louverture's fight for freedom. Now be more specific. What did Spain offer to help Louverture and his troops?,TRUE, +Spain was at war with France at the time,That's true! Now be more specific. How could Spain help Louverture defeat France?,TRUE, +Spain was in conflict with France during that period,That's true! Now be more specific. How could Spain help Louverture defeat France?,TRUE, +of the Spanish Armada,Try clearing your response and starting again. Why did Louverture want to partner with Spain? Check that your response only uses information from the text.,TRUE, +he believed that they would support his figt for freedom against the French colonists,It's true that Spain supported Louverture's fight for freedom. Now be more specific. What did Spain offer to help Louverture and his troops?,TRUE, +he was convinced that they would back his struggle for independence against the French settlers.,That's true! Now be more specific. What did Spain offer Louverture to help him defeat France?,TRUE, +he believed that they would suport his fight for freedom against the French colonists,It's true that Spain supported Louverture's fight for freedom. Now be more specific. What did Spain offer to help Louverture and his troops?,TRUE, +he was certain that they would back his struggle for independence against the French settlers.,That's true! Now be more specific. What did Spain offer Louverture to help him defeat France?,TRUE, +Sain was at war with France at the time,That's true! Now be more specific. How could Spain help Louverture defeat France?,TRUE, +Spain was at war with Fance at the time,That's true! Now be more specific. How could Spain help Louverture defeat France?,TRUE, +"Spain was an ally of France's enemy, Great Britain",That's not quite right. The text explains why Louverture allied with Spain. Reread the section about Spain's promises to Louverture and his troops. What did Spain offer that would help Louverture fight the French?,FALSE, +"the country of Spain was a friend of France's adversary, the United Kingdom.",That's not quite right. The text explains why Louverture allied with Spain. Reread the section about Spain's promises to Louverture and his troops. What did Spain offer that would help Louverture fight the French?,FALSE, +"Spain was a partner of France's foe, Great Britain.",That's not quite right. The text doesn't mention Great Britain. Reread the text and focus on why Louverture would want to ally with Spain. What did Spain offer him?,FALSE, +"Spain was a friend of France's adversary, Great Britain.",That's not quite right. The text explains why Louverture allied with Spain. Reread the section about Spain's promises to Louverture and his troops. What did Spain offer that would help Louverture fight the French?,FALSE, +he wanted to protect the interests of the free people of color and the slaves,It's true that Louverture wanted to protect the interests of the free people of color and the slaves. Now add more information. What did Spain offer to help Louverture and his troops?,TRUE, +"Spain was an ally of Frane's enemy, Great Britain",That's not quite right. The text explains why Louverture allied with Spain. Reread the section about Spain's promises to Louverture and his troops. What did Spain offer that would help Louverture fight France?,FALSE, +"Spai was an ally of France's enemy, Great Britain",That's not quite right. The text explains why Louverture allied with Spain. Reread the section about Spain's promises to Louverture and his troops. What did Spain offer that would help Louverture fight the French?,FALSE, +he saw them as a potential ally against the French colonists in Saint-Domingue,That's true! Now be more specific. What did Spain offer Louverture to help him defeat the French?,TRUE, +he wanted to protect the rights and freedoms of the enslaved people,It's true that Louverture wanted to end slavery. Now add more information. How did he think a partnership with Spain would help him achieve this goal?,TRUE, +he wanted to protect the rights and freedoms of the ensaved people,It's true that Louverture was fighting to end slavery. Now add more information. How did he think a partnership with Spain would help him achieve this goal?,TRUE, +"the nation of Spain supported Great Britain's adversary, France",That's not quite right. The text says that Spain and France had a shared goal: weakening French power in Hispaniola. Reread the text and focus on what Spain offered Louverture to help him fight the French.,FALSE,wrong_fact +he saw them as a potential ally against the French colonists in Sain-Domingue,That's true! Now be more specific. What did Spain offer Louverture to help him defeat the French?,TRUE, +he saw them as a potential ally against the Frnch colonists in Saint-Domingue,That's true! Now be more specific. What did Spain offer Louverture to help him defeat the French?,TRUE, +France was unwilling to give up control of Saint-Domingue without a fight,That's true! Now be more specific. What did Spain offer Louverture to help him defeat France?,TRUE, +France refused to give up control of Saint-Domingu,That's true! Now be more specific. What did Spain offer Louverture to help him defeat France?,TRUE, +he saw them as a potential ally against the French colonists in Saint-Dominge,That's true! Now be more specific. What did Spain offer Louverture to help him defeat the French?,TRUE, +of the French Revolution,That's not quite right. The sentence should explain why Toussaint Louverture formed an alliance with Spain in 1793. Look back at the text to find information about what Spain offered Louverture that would help him in his fight against France.,FALSE,tone +he was the leader of the Haitian Revolution against the Frnch colonial rule,"That's true, but it doesn't explain why he formed an alliance with Spain. The text explains what Spain offered Louverture to help him fight the French. Try again, focusing on what Spain offered Louverture.",FALSE,verbose +he was the leader of the Haitian Revoltion against the French colonial rule,That's true! Now be more specific. What did Spain offer Louverture to help him defeat France?,TRUE, +he wanted to protect the interests of the free pople of color and the slaves,It's true that Louverture wanted to protect the enslaved people and free people of color. Now add more information. What did Spain offer to help Louverture and his troops?,TRUE, +he was the leader of the Haitian Revolution against the French colonial rule,"That's true, but it doesn't explain why he formed an alliance with Spain. The text explains what Spain offered Louverture that made the alliance beneficial. Try again, focusing on what Spain offered Louverture.",FALSE,verbose +he wanted to protect the rights and freedoms of the enslaved peope,It's true that Louverture wanted to end slavery. Now add more information. How did he think a partnership with Spain would help him achieve this goal?,TRUE, +he was a military leader who was born a slave,"That's true, but it doesn't explain why he formed an alliance with Spain. Reread the text and focus on why Louverture needed Spain's help.",TRUE, +la la la la,Try clearing your response and starting again. Why did Louverture want to partner with Spain? Check that your response only uses information from the text.,TRUE, +Spain was an enemy of France and Toussaint Louverture needed all the help he could get,That's true! Now be more specific. What kind of help did Spain offer Louverture?,TRUE, +he believed that Spain would be more supportive of the abolition of savery in Saint-Domingue,It's true that Louverture wanted to end slavery. Now add more information. What did Spain offer to help Louverture and his troops fight the French?,TRUE, +Spain was an enemy of France adn Toussaint Louverture needed all the help he could get,That's true! Now be more specific. How could Spain help Louverture defeat France?,TRUE, +in order to defeat them.,You have the right idea that Louverture wanted to defeat the French. Now add more information. What did Spain offer to help him achieve this goal?,TRUE, +he believed that Spain would be more supportive of the abolition of slavery in Sant-Domingue,It's true that Spain's support for abolition was important to Louverture. Now add more information. What did Spain offer Louverture to help him defeat France?,TRUE, +he believed that Spain would be more supportive of the abolition of slavery in Saint-Domingue,It's true that Louverture wanted to end slavery. Now add more information. What did Spain offer to help Louverture and his troops fight the French?,TRUE, +he believed that Spain would be more supportive of the abolition of slavery in Saint-Dominue,It's true that Spain's support for abolition was important to Louverture. Now add more information. What did Spain offer Louverture to help him defeat France?,TRUE, +to defeat them.,You have the right idea that Louverture wanted to defeat the French. Now add more information. What did Spain offer to help him achieve this goal?,TRUE, +Spain was an enemy of France and Toussaint Loverture needed all the help he could get,That's true! Now be more specific. How could Spain help Louverture defeat France?,TRUE, +he believed that Sain would be more supportive of the abolition of slavery in Saint-Domingue,"It's true that Spain's support might have helped end slavery. However, the text focuses on what Spain offered Louverture to help him fight the French. Can you explain what resources or support Spain offered Louverture to help him in his fight against France?",FALSE,unhelpful +Spain was an enemy of Franc and Toussaint Louverture needed all the help he could get,That's true! Now be more specific. What kind of help did Spain offer Louverture?,TRUE, +Spai was an enemy of France and Toussaint Louverture needed all the help he could get,That's true! Now be more specific. What kind of help did Spain offer Louverture?,TRUE, +he thought that Spain would support his cause for independence,That's true! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he believed that Spain would endorse his quest for autonomy.,It's true that Louverture wanted autonomy. Now add more information. What did Spain offer to help Louverture achieve his goals?,TRUE, +the French were not willing to give up their hold on the island,That's true! Now be more specific. What did Spain offer Louverture to help him defeat France?,TRUE, +Spain was against France,That's true! Now be more specific. How could Spain help Louverture defeat France?,TRUE, +Spain was opposed to France.,That's true! Now be more specific. How could Spain help Louverture defeat France?,TRUE, +Spain opposed France.,That's true! Now be more specific. How could Spain help Louverture defeat France?,TRUE, +Spain opposed France,That's true! Now be more specific. How could Spain help Louverture defeat France?,TRUE, +the Frech were not willing to give up their hold on the island,That's true! Now be more specific. What did Spain offer Louverture to help him defeat France?,TRUE, +he believed that Spain would back his quest for independence.,"That's true, but it's not specific enough. What did Spain offer Louverture that would help him achieve independence?",FALSE,tone +he thought that Sain would support his cause for independence,That's true! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +Spain was against Fance,That's true! Now be more specific. How could Spain help Louverture defeat France?,TRUE, +he believed that Spain would back his efforts for autonomy.,"That's true, but it's not specific enough. How did Spain's support help Louverture in his fight against France?",FALSE,tone +the French were not willing to give up their hold on the islnd,That's true! Now be more specific. What did Spain offer Louverture to help him defeat France?,TRUE, +Spin was against France,That's true! Now be more specific. How could Spain help Louverture defeat France?,TRUE, +the French revolutionary government had promised to reestablish slvery in the French colonies,That's incorrect. The text states that France ended slavery in its colonies in 1794. Reread the text and focus on why Louverture allied with Spain in 1793.,FALSE,tone +he thought that Spain would suport his cause for independence,It's true that Spain supported Louverture's cause. Now be more specific. What did Spain offer to help Louverture and his troops?,TRUE, +the French revolutionary government had promised to reestablish slavery in the French colonies,That's incorrect. The text states that France ended slavery in its colonies in 1794. Reread the text and focus on why Louverture allied with Spain in 1793.,FALSE, +the French revolutionar government had promised to reestablish slavery in the French colonies,That's incorrect. The text states that France ended slavery in its colonies in 1794. Reread the text and focus on why Louverture allied with Spain in 1793.,FALSE, +the Frech revolutionary government had promised to reestablish slavery in the Frech colonies,That's incorrect. The text states that France ended slavery in its colonies in 1794. Reread the text and focus on why Louverture allied with Spain in 1793.,FALSE, +Spain was an enemy of France and Toussaint Louverture needed all he help he could get,That's true! Now be more specific. What kind of help did Spain offer Louverture?,TRUE, +he believed that Spain would back his claim for autonomy.,"That's true, but it's not the main reason Louverture formed an alliance with Spain. The text mentions other key reasons. Can you find them and explain why Louverture chose to partner with Spain?",FALSE,unhelpful +he believed it would help him end enlavement,It's true that Louverture wanted to end slavery. Now add more information. How did he think a partnership with Spain would help him achieve this goal?,TRUE, +he believed it would help him end enslavement,It's true that Louverture wanted to end slavery. Now add more information. How did he think a partnership with Spain would help him achieve this goal?,TRUE, +he was a slave in Haiti and he wanted to fight for his freedom and the freedom of his people,It's true that Louverture fought for freedom. Now add more information. How did he think a partnership with Spain would help him achieve this goal?,TRUE, +he was a slave in Haiti and he wanted to fght for his freedom and the freedom of his people,It's true that Louverture fought for freedom. Now add more information. How did he think a partnership with Spain would help him achieve this goal?,TRUE, +he could not defeat the French army,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +he could not defeat the Frenh army,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture?,TRUE, +from slavery,It's true that Spain promised freedom to enslaved people who joined their army. Now be more specific. What did Louverture hope to achieve by allying with Spain?,FALSE,gives_answer +he was a free black man who had been a slave,"That's true, but it doesn't explain why he formed an alliance with Spain. Reread the text and focus on what Spain offered Louverture that would help him in his fight against France.",FALSE,tone +Frace was not willing to abolish slavery,That's true! Now be more specific. What did Spain offer Louverture to help him defeat France?,TRUE, +he believed they would support the abolition of savery,It's true that Spain's alliance could help end slavery. Now add more information. What did Spain offer Louverture to help him defeat France?,TRUE, +he was a slave in Haiti and he wanted to fight for his freedom and the freedom of his pople,It's true that Louverture fought for freedom. Now add more information. How did he think a partnership with Spain would help him achieve this goal?,TRUE, +France was not willing to abolish slavery,"That's true; France was initially unwilling to abolish slavery. However, this fact doesn't explain why Louverture allied with Spain. Consider what resources or support Spain offered that would help Louverture in his fight against France.",FALSE,gives_answer +he believed they would support the abolition of slavery,It's true that Spain's promise to free enslaved people who joined their army was important to Louverture. Now add more information. What resources did Spain offer that would help Louverture fight the French?,FALSE,gives_answer +France was not willing to abolish savery,"It's true that France was initially unwilling to abolish slavery. However, that's not why Louverture allied with Spain. The text explains what Spain offered Louverture to help him fight the French. Try again, focusing on what Spain offered Louverture.",FALSE,unhelpful +Spain supported the abolition of slavery,"It's true that Spain's alliance with Louverture eventually led to the abolition of slavery for some enslaved people. However, the prompt asks for the reason Toussaint Louverture formed an alliance with Spain in 1793. What did Spain offer Louverture that made him want to form this alliance?",FALSE,unhelpful +Spain supported the abolition of savery,"It's true that Spain's alliance with Louverture eventually led to the abolition of slavery for some enslaved people. However, the prompt asks for the reason Toussaint Louverture formed an alliance with Spain in 1793. Please re-read the text and focus on what Spain offered Louverture at that time to help him in his fight against France.",FALSE,unhelpful +"Spain could help him win contrl of several important cities, ports, and territories","That was a result of Louverture's partnership with Spain. Clear your response and start again. This time, write about a reason why Louverture wanted to work with Spain.",TRUE, +he needed to get rid of the Fench,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture to help him defeat the French?,TRUE, +he needed to get rid of the French,You have the right idea! Now be more specific. What kind of support did Spain offer to Louverture to help him defeat the French?,TRUE, +Spain was a major colonial power in the Caribbean,"That's true, but it doesn't explain why Louverture formed an alliance with Spain. What did Spain offer Louverture that made the alliance beneficial to him?",TRUE, +"Spain could help him win control of several important cities, ports, and territries","That was a result of Louverture's partnership with Spain. Clear your response and start again. This time, write about a reason why Louverture wanted to work with Spain.",TRUE, +he wanted to drive the French out of Saint-Domingue,That's true! Now be more specific. How could Spain help Louverture defeat France?,TRUE, +France was still a monarchy and Spai was a republic,"That's interesting, but the text doesn't mention the type of government in either country as a reason for the alliance. Can you find a reason in the text that explains why Louverture allied with Spain?",FALSE,bonkers +Frane was still a monarchy and Spain was a republic,"That's interesting information about the governments of France and Spain. However, it doesn't explain why Louverture formed an alliance with Spain. Reread the text and focus on what Spain offered Louverture that would help him in his fight against France.",FALSE,bonkers +"Spai could help him win control of several important cities, ports, and territories","That was a result of Louverture's partnership with Spain. Clear your response and start again. This time, write about a reason why Louverture wanted to work with Spain.",TRUE, +"he was able to gain contol of several important cities, ports, and territories","That was a result of Louverture's partnership with Spain. Clear your response and start again. This time, write about a reason why Louverture wanted to work with Spain.",TRUE, +he wanted to,"You have the right idea, but you need to add more information. Why did Louverture want to partner with Spain? What did Spain offer him?",FALSE,two_directions +"he was able to gain control of several important cities, ports, and territories","That was a result of Louverture's partnership with Spain. Clear your response and start again. This time, write about a reason why Louverture wanted to work with Spain.",TRUE, +Spin was a major colonial power in the Caribbean,That's not quite right. The text focuses on Spain's role in helping Toussaint Louverture fight the French. Try rereading the text and focusing on what Spain offered Louverture to help him in his fight.,TRUE, +he wanted to drive the Frech out of Saint-Domingue,That's true! Now be more specific. How could Spain help Louverture defeat France?,TRUE, +France was still a monarchy and Spain was a republic,"That's interesting, but the text doesn't mention the type of government in either country. Try clearing your response and starting again. This time, write about why Louverture wanted to partner with Spain. How did he think Spain could help him?",FALSE,bonkers +he wanted to drive the French out of Sint-Domingue,That's true! Now be more specific. How could Spain help Louverture defeat France?,TRUE, +he wanted to drive the French out of Saint-Dmingue,That's true! Now be more specific. How could Spain help Louverture defeat France?,TRUE, +"he wanted to gain conrol of several important cities, ports, and territories","That was a result of Louverture's partnership with Spain. Clear your response and start again. This time, write about a reason why Louverture wanted to work with Spain.",TRUE, +"he wanted to gain control of several important cities, ports and territories","That was a result of Louverture's partnership with Spain. Clear your response and start again. This time, write about a reason why Louverture wanted to work with Spain.",TRUE, +"he wanted to gain control of several important cities, ports, and territorie","That was a result of Louverture's partnership with Spain. Clear your response and start again. This time, write about a reason why Louverture wanted to work with Spain.",TRUE, +"he wanted to gain control of several important cities, ports, and territories","That was a result of Louverture's partnership with Spain. Clear your response and start again. This time, write about a reason why Louverture wanted to work with Spain.",TRUE, +"he wanted to gain contol of several important cities, ports and territories","That was a result of Louverture's partnership with Spain. Clear your response and start again. This time, write about a reason why Louverture wanted to work with Spain.",TRUE, +"it would help him to win control of important cities, ports, and territoies","That was a result of Louverture's partnership with Spain. Clear your response and start again. This time, write about a reason why Louverture wanted to work with Spain.",TRUE, +"it would help him to win conrol of important cities, ports, and territories","That was a result of Louverture's partnership with Spain. Clear your response and start again. This time, write about a reason why Louverture wanted to work with Spain.",TRUE, +"it would help him to win control of important cities, ports, and territories","That was a result of Louverture's partnership with Spain. Clear your response and start again. This time, write about a reason why Louverture wanted to work with Spain.",TRUE, +"he wanted to gain control of several important cities, ports and territores","That was a result of Louverture's partnership with Spain. Clear your response and start again. This time, write about a reason why Louverture wanted to work with Spain.",TRUE, +"he wanted to increase his military power and to gain control of several important cities, ports, and territories","That was a result of Louverture's partnership with Spain. Clear your response and start again. This time, write about a reason why Louverture wanted to work with Spain.",TRUE, +"he wanted to increase his military power and to gain control of several important cities, ports, and terriories","That was a result of Louverture's partnership with Spain. Clear your response and start again. This time, write about a reason why Louverture wanted to work with Spain.",TRUE, +"he wanted to increase his military power and to gain cotrol of several important cities, ports, and territories","That was a result of Louverture's partnership with Spain. Clear your response and start again. This time, write about a reason why Louverture wanted to work with Spain.",TRUE, +he believed it would help him achieve his goals,It's true that Louverture believed a partnership with Spain would help him achieve his goals. Now add more information. What did Spain offer that would make it possible to achieve his goals?,TRUE, +"the Frech had promised to free the slaves, but had not kept their promise","That's true, but that's not why Louverture formed an alliance with Spain. The text explains why Louverture chose to work with Spain. Reread the section about why Louverture formed an alliance with Spain and try again.",FALSE,verbose +"the French had promised to free the slaves, but had not kept their promise","That's true, but that's not why Louverture formed an alliance with Spain. The text explains why Louverture chose to work with Spain. Try to use information from the text to explain why Louverture formed an alliance with Spain in 1793.",FALSE,verbose +he was a very pragmatic and shrewd leader.,That's true! Now add more information. What did Spain offer Louverture to help him defeat France?,FALSE,misacknowledgement +he believed that Spai would help him to achieve his goals.,It's true that Louverture believed Spain could help him achieve his goals. Now add more information. What did Spain offer that would make it possible to achieve his goals?,TRUE, +he believed that Spain would help him to achieve his goals.,It's true that Louverture believed Spain could help him achieve his goals. Now add more information. What did Spain offer that would make it possible to achieve his goals?,TRUE, +the French Revolution had caused instability in Saint-Domingue,"That's true, but it doesn't explain why Louverture formed an alliance with Spain. The text explains what Spain offered Louverture to help him fight the French. Try again, focusing on what Spain offered Louverture.",FALSE,verbose +he felt that the Spanish were more sympathetic to the plight of the enslaved population in Sain-Domingue,It's true that Spain's offer of freedom to enslaved people who joined their army was appealing to Louverture. Now add more information. What did Spain offer that would help Louverture defeat the French?,FALSE,gives_answer +the French Revolution had caused instability in Saint-Domngue,"That's true, but it doesn't explain why Louverture formed an alliance with Spain. The text explains what Spain offered Louverture to help him fight the French. Try again, focusing on what Spain offered Louverture.",FALSE,verbose +he felt that the Spanish were more sympathetic to the plight of the enslaved population in Saint-Domingue,It's true that Spain's offer of freedom to enslaved people who joined their army was appealing. Now add more information. What did Spain offer that would help Louverture and his troops fight the French?,FALSE,gives_answer +the French Revolutin had caused instability in Saint-Domingue,"That's true, but it doesn't explain why Louverture formed an alliance with Spain. The text explains what Spain offered Louverture to help him fight the French. Try again, focusing on what Spain offered Louverture.",FALSE,verbose +the French Revolution had caused instability in Sint-Domingue,"That's true, but it doesn't explain why Louverture formed an alliance with Spain. The text explains what Spain offered Louverture to help him fight the French. Try again, focusing on what Spain offered Louverture.",FALSE,verbose +he felt that the Spanish were more sympathetic to the plight of the enslaved population in Saint-Domigue,It's true that Spain's offer of freedom to enslaved people who joined their army was appealing to Louverture. Now add more information. What resources did Spain offer that would help Louverture fight the French?,FALSE,gives_answer +he wanted to get rid of the white French colonists,That's not quite right. The text focuses on Louverture's reasons for allying with Spain to fight the French. What did Spain offer that would help him in this fight?,TRUE, +he felt that the Sanish were more sympathetic to the plight of the enslaved population in Saint-Domingue,"That's true, but the text doesn't say that. The text mentions what Spain offered Louverture in exchange for his alliance. Can you use that information to explain why Louverture formed an alliance with Spain?",FALSE,unhelpful +he wanted to get rid of the white Fench colonists,That's not quite right. The text focuses on Louverture's reasons for allying with Spain to fight the French. What did Spain offer that would help him in this fight?,TRUE, +he felt that the Spanish were more sympathetic to the plight of the enlaved population in Saint-Domingue,It's true that Spain's offer of freedom to enslaved people who joined their army was appealing to Louverture. Now add more information. What resources did Spain offer that would help Louverture fight the French?,FALSE,gives_answer +he felt that the Spanish were more sympathetic to the plight of the enslaved populaton in Saint-Domingue,It's true that Spain's offer of freedom to enslaved people who joined their army was appealing. Now add more information. What did Spain offer that would help Louverture defeat the French?,FALSE,gives_answer +Spin supported the abolition of slavery,That's true! Now be more specific. How did Spain's offer help Louverture and his troops fight against France?,FALSE,unhelpful +"Spain could help him win control of several important cities, ports, and territories","That was a result of Louverture's partnership with Spain. Clear your response and start again. This time, write about a reason why Louverture wanted to work with Spain.",TRUE, +"he was committed to the abolition of slavery, and he believed that Spain was more likely to support his cause than France.","It's true that Louverture was committed to ending slavery. However, your response doesn't explain why he chose to ally with Spain at that particular moment. The text explains what Spain offered Louverture that made an alliance appealing. Try to use that information to answer the question.",FALSE,verbose +he wanted to free Saint-Domingue from France,That's true! Now be more specific. How could a partnership with Spain help Louverture free Saint-Domingue from France?,TRUE, +he aimed to liberate Saint-Domingue from the control of France.,That's true! Now be more specific. How could a partnership with Spain help Louverture achieve this goal?,TRUE, +he desired to liberate Saint-Domingue from French control.,That's true! Now be more specific. How could a partnership with Spain help Louverture achieve this goal?,TRUE, +he desired to liberate Saint-Domingue from French rule.,It's true that Louverture wanted to free Saint-Domingue. Now add more information. How did he think a partnership with Spain would help him achieve this goal?,TRUE, +he wanted to protect the planters and the whites in Sait-Domingue,That's not quite right. The text focuses on Louverture's efforts to end slavery and fight against France. Reread the text and focus on what Spain offered Louverture to help him achieve his goals.,TRUE, +"he was able to gain control of several important cities, ports, and territries","That was a result of Louverture's partnership with Spain. Clear your response and start again. This time, write about a reason why Louverture wanted to work with Spain.",TRUE, +"he was committed to the abolition of slavery, and he believed that Spain was more likely to support his cause than Frane.",It's true that Louverture was committed to ending slavery. Now add more information. How did he think a partnership with Spain would help him achieve this goal?,TRUE, +he wanted too free Saint-Domingue from France,It's true that Louverture wanted to free Saint-Domingue. Now add more information. How did he think a partnership with Spain would help him achieve this goal?,TRUE, +he wanted to protect the planters and the whites in Saint-Dominge,That's not quite right. The text focuses on Louverture's efforts to fight for the freedom of enslaved people. Reread the text and focus on why Louverture would want to partner with Spain to help achieve his goals.,TRUE, +he wanted to free Saint-Domingue form France,It's true that Louverture wanted to free Saint-Domingue. Now add more information. How did he think a partnership with Spain would help him achieve this goal?,TRUE, +he wanted to free Saint-Domingue from Frane,It's true that Louverture wanted to free Saint-Domingue. Now add more information. How did he think a partnership with Spain would help him achieve this goal?,TRUE, +"he was committed to the abolition of slavery, and he believed that Span was more likely to support his cause than France.","It's true that Louverture was committed to ending slavery. However, your response doesn't explain why he chose to ally with Spain at that particular moment. The text explains what Spain offered Louverture that made an alliance appealing. Try to use that information to answer the question.",FALSE,verbose +he wanted to protect the planters and the whites in Saint-Domingue,That's not quite right. The text focuses on Louverture's efforts to fight for the freedom of enslaved people. Reread the text and focus on why Louverture would want to partner with Spain to help achieve his goals.,TRUE, +he wanted to free Saint-Domngue from France,It's true that Louverture wanted to free Saint-Domingue from France. Now add more information. What did Spain offer to help Louverture achieve this goal?,TRUE, +"he was committed to the abolition of slaery, and he believed that Spain was more likely to support his cause than France.","It's true that Louverture was committed to ending slavery. However, your response doesn't explain why he chose to ally with Spain at that particular moment. The text explains what Spain offered Louverture that made an alliance appealing. Try to use that information to answer the question.",FALSE,verbose +he wanted to free Sant-Domingue from France,It's true that Louverture wanted to free Saint-Domingue from France. Now add more information. How did he think a partnership with Spain would help him achieve this goal?,TRUE, +the Frenc had abolished slavery in 1794,"That's true, but that happened after Louverture formed an alliance with Spain. Why did he form the alliance *before* that event happened?",TRUE, +the Frenc revolutionaries abolished slavery in 1794,"That's true, but that's not why Louverture formed an alliance with Spain in 1793. The text explains why Louverture chose to work with Spain at that time. Try reading that section again and focusing on what Spain offered Louverture.",TRUE, +the French revolutionaries abolished slavery in 1794,"That's true, but that happened after Louverture formed an alliance with Spain. The question asks why he formed the alliance *in 1793*. What did Spain offer that made the alliance attractive to Louverture at that time?",TRUE, +the French revoltionaries abolished slavery in 1794,"That's true, but that happened after Louverture formed an alliance with Spain. The question asks why he formed the alliance *in 1793*. What did Spain offer that made the alliance attractive to Louverture at that time?",TRUE, +the French had abolished slvery in 1794,"It's true that France abolished slavery in 1794, but that's not why Louverture allied with Spain in 1793. Try again. What did Spain offer Louverture that made him want to form an alliance?",TRUE, +the French had abolished slavery in 1794,"It's true that France abolished slavery in 1794, but that's not why Louverture allied with Spain in 1793. Try again. What did Spain offer Louverture that made him want to form an alliance?",TRUE, +the French revolutionaries abolished slvery in 1794,"That's true, but that's not why Louverture formed an alliance with Spain in 1793. The text explains why Louverture chose to ally with Spain earlier than 1794. Reread the section about Louverture's alliance with Spain and try again.",TRUE, +asdfkasdjflksadjf;sadjfkasdfkjsndfkjdpo asdfklasjfpoijwaerf,Try clearing your response and starting again. Why did Louverture want to partner with Spain? Check that your response only uses information from the text.,TRUE, +the Frnch Revolution had caused instability in Saint-Domingue,"That's true, but this response doesn't explain why Louverture formed an alliance with Spain. The text explains what Spain offered Louverture in exchange for his alliance. Try again, focusing on what Spain offered Louverture.",FALSE,verbose +idk idk idk idk idk idk idk,Try clearing your response and starting again. Why did Louverture want to partner with Spain? Check that your response only uses information from the text.,TRUE, +France refused to give up control of Saint-Domingue,That's true! Now be more specific. What did Spain offer Louverture to help him defeat France?,TRUE, \ No newline at end of file diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/data/underage_soldiers_because.csv b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/data/underage_soldiers_because.csv new file mode 100644 index 00000000000..2940b9be0f7 --- /dev/null +++ b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/data/underage_soldiers_because.csv @@ -0,0 +1,199 @@ +entry,feedback,optimal,rule +boys faced pressure.,Some boys did feel pressure to enlist in the British Army---that's true! Now add more specific information to your response. Why did some boys feel pressure to volunteer for the army?,TRUE, +it made young boys feel like they were pressured,It's true that many boys felt pressure to enlist in the British Army---that's true! Now add more specific information to your response. Why did some boys feel pressure to volunteer for the army?,FALSE,misworded +stress from figures made them feel pressure,Some boys did feel pressure to enlist in the British Army---that's true! Now add more specific information to your response. Why did some boys feel pressure to volunteer for the army?,TRUE, +boys in Britain were pressured to enlist to serve their country.,Some boys did feel pressure to enlist in the British Army---that's true! Now add more specific information to your response. Why did some boys feel pressure to volunteer for the army?,TRUE, +boys in Britain faced pressure to enlist in order to serve their country.,Some boys did feel pressure to enlist in the British Army---that's true! Now add more specific information to your response. Why did some boys feel pressure to volunteer for the army?,TRUE, +there was a pressure for young men during that time.,Some boys did feel pressure to enlist in the British Army---that's true! Now add more specific information to your response. Why did some boys feel pressure to volunteer for the army?,TRUE, +the pressure of their society made them feel the need to do so.,Some boys did feel pressure to enlist in the British Army---that's true! Now add more specific information to your response. Why did some boys feel pressure to volunteer for the army?,TRUE, +the pressure of young men was very strong at the time.,Some boys did feel pressure to enlist in the British Army---that's true! Now add more specific information to your response. Why did some boys feel pressure to volunteer for the army?,TRUE, +British boys were pressured to enlist in order to serve their country.,Some boys did feel pressure to enlist in the British Army---that's true! Now add more specific information to your response. Why did some boys feel pressure to volunteer for the army?,TRUE, +there was a sense of pressure among young men at the time.,Some boys did feel pressure to enlist in the British Army---that's true! Now add more specific information to your response. Why did some boys feel pressure to volunteer for the army?,TRUE, +the British Army needed more men to join the fright for them.,It's true that the British Army needed more soldiers during WWI—that's helpful historical context. Write about a specific type of propaganda that convinced some young boys to join the army. Why did some boys feel inspired to join the British Army?,TRUE, +many young men of the time felt a strong pressure.,Some boys did feel pressure to enlist in the British Army---that's true! Now add more specific information to your response. Why did some boys feel pressure to volunteer for the army?,TRUE, +British boys were pressured to serve,Some boys did feel pressure to enlist in the British Army---that's true! Now add more specific information to your response. Why did some boys feel pressure to volunteer for the army?,TRUE, +some young men at the time had a strong pressure and motivation.,Some boys did feel pressure to enlist in the British Army---that's true! Now add more specific information to your response. Why did some boys feel pressure to volunteer for the army?,TRUE, +there was a powerful pressure among young men at the time.,Some boys did feel pressure to enlist in the British Army---that's true! Now add more specific information to your response. Why did some boys feel pressure to volunteer for the army?,TRUE, +British army needed more men on the front line.,It's true that the British Army needed more soldiers during WWI—that's helpful historical context. Write about a specific type of propaganda that convinced some young boys to join the army. Why did some boys feel inspired to join the British Army?,TRUE, +there was a strong pressures for young men at the time.,Some boys did feel pressure to enlist in the British Army---that's true! Now add more specific information to your response. Why did some boys feel pressure to volunteer for the army?,TRUE, +the pressure of young men was at an all-time high.,Some boys did feel pressure to enlist in the British Army---that's true! Now add more specific information to your response. Why did some boys feel pressure to volunteer for the army?,TRUE, +Britain needed to recruit volunteers for the war.,It's true that the British Army needed more soldiers during WWI—that's helpful historical context. Write about a specific type of propaganda that convinced some young boys to join the army. Why did some boys feel inspired to join the British Army?,TRUE, +Britain's boys faced pressure to enlist in order to serve their country.,Some boys did feel pressure to enlist in the British Army---that's true! Now add more specific information to your response. Why did some boys feel pressure to volunteer for the army?,TRUE, +boys in Briain faced pressure to enlist in order to serve their country.,Some boys did feel pressure to enlist in the British Army---that's true! Now add more specific information to your response. Why did some boys feel pressure to volunteer for the army?,TRUE, +some were motivated and wanted to help their country out because they were pressured to help their country.,Some boys did feel pressure to enlist in the British Army---that's true! Now add more specific information to your response. Why did some boys feel pressure to volunteer for the army?,TRUE, +they didn't have enough volunteers.,It's true that the British Army needed more soldiers during WWI—that's helpful historical context. Write about a specific type of propaganda that convinced some young boys to join the army. Why did some boys feel inspired to join the British Army?,TRUE, +Great Britain was going to war and the British Army needed more men to join the fight.,It's true that the British Army needed more soldiers during WWI—that's helpful historical context. Write about a specific type of propaganda that convinced some young boys to join the army. Why did some boys feel inspired to join the British Army?,TRUE, +there was intense pressure for young men in the time.,Some boys did feel pressure to enlist in the British Army---that's true! Now add more specific information to your response. Why did some boys feel pressure to volunteer for the army?,TRUE, +their country needed them in the military.,It's true that the British Army needed more soldiers during WWI—that's helpful historical context. Write about a specific type of propaganda that convinced some young boys to join the army. Why did some boys feel inspired to join the British Army?,TRUE, +there was a very strong pressure for young men at the time.,Some boys did feel pressure to enlist in the British Army---that's true! Now add more specific information to your response. Why did some boys feel pressure to volunteer for the army?,TRUE, +the British army was lacking soldiers.,It's true that the British Army needed more soldiers during WWI—that's helpful historical context. Write about a specific type of propaganda that convinced some young boys to join the army. Why did some boys feel inspired to join the British Army?,TRUE, +they were pressured to enlist in their everyday lives due to the British Army needing more men to join the fight.,Some boys did feel pressure to enlist in the British Army---that's true! Now add more specific information to your response. Why did some boys feel pressure to volunteer for the army?,TRUE, +there was a strong feeling of pressure among young men at the time.,Some boys did feel pressure to enlist in the British Army---that's true! Now add more specific information to your response. Why did some boys feel pressure to volunteer for the army?,TRUE, +The British needed more help.,It's true that the British Army needed more soldiers during WWI—that's helpful historical context. Write about a specific type of propaganda that convinced some young boys to join the army. Why did some boys feel inspired to join the British Army?,TRUE, +the British Army needed more men to join the fight motivated by beliefs and pride.,"It's true that the British Army needed more soldiers during WWI—that's helpful historical context. However, this response doesn't explain why the boys themselves were motivated to join. The text mentions several reasons. Can you add details from the text to explain why the boys felt pride and what beliefs motivated them to join the fight?",FALSE,"two_directions,tone" +Britain needed soldiers for the war.,It's true that the British Army needed more soldiers during WWI—that's helpful historical context. Write about a specific type of propaganda that convinced some young boys to join the army. Why did some boys feel inspired to join the British Army?,TRUE, +Britain needed people to enlist in the war.,It's true that the British Army needed more soldiers during WWI—that's helpful historical context. Write about a specific type of propaganda that convinced some young boys to join the army. Why did some boys feel inspired to join the British Army?,TRUE, +the country needed the soldiers to fight.,It's true that the British Army needed more soldiers during WWI—that's helpful historical context. Write about a specific type of propaganda that convinced some young boys to join the army. Why did some boys feel inspired to join the British Army?,TRUE, +They needed more men in there army to fight.,It's true that the British Army needed more soldiers during WWI—that's helpful historical context. Write about a specific type of propaganda that convinced some young boys to join the army. Why did some boys feel inspired to join the British Army?,TRUE, +Britain needed people to volunteer for the war.,It's true that the British Army needed more soldiers during WWI—that's helpful historical context. Write about a specific type of propaganda that convinced some young boys to join the army. Why did some boys feel inspired to join the British Army?,TRUE, +their country needed them to fight on their behalf.,It's true that the British Army needed more soldiers during WWI—that's helpful historical context. Write about a specific type of propaganda that convinced some young boys to join the army. Why did some boys feel inspired to join the British Army?,TRUE, +Britain needed volunteers for the war effort.,It's true that the British Army needed more soldiers during WWI—that's helpful historical context. Write about a specific type of propaganda that convinced some young boys to join the army. Why did some boys feel inspired to join the British Army?,TRUE, +Britain needed them to participate.,It's true that the British Army needed more soldiers during WWI—that's helpful historical context. Write about a specific type of propaganda that convinced some young boys to join the army. Why did some boys feel inspired to join the British Army?,TRUE, +Britain needed them to help fight.,It's true that the British Army needed more soldiers during WWI—that's helpful historical context. Write about a specific type of propaganda that convinced some young boys to join the army. Why did some boys feel inspired to join the British Army?,TRUE, +Britain needed the troops to fight.,It's true that the British Army needed more soldiers during WWI—that's helpful historical context. Write about a specific type of propaganda that convinced some young boys to join the army. Why did some boys feel inspired to join the British Army?,TRUE, +they bought into the war propaganda.,It's true that propaganda inspired some young boys to join the army. Now add another detail from the text. What kind of propaganda was used to convince young boys to join the army?,TRUE, +the Britian military needed these troops to fight.,It's true that the British Army needed more soldiers during WWI—that's helpful historical context. Write about a specific type of propaganda that convinced some young boys to join the army. Why did some boys feel inspired to join the British Army?,TRUE, +they were shown propaganda from an early age.,It's true that propaganda inspired some young boys to join the army. Now add another detail from the text. What kind of propaganda was used to convince young boys to join the army?,TRUE, +they saw lots of propaganda in support of the war,It's true that propaganda inspired some young boys to join the army. Now add another detail from the text. What kind of propaganda was used to convince young boys to join the army?,TRUE, +they were influenced by the propaganda that was surrounding them.,It's true that propaganda inspired some young boys to join the army. Now add another detail from the text. What kind of propaganda was used to convince young boys to join the army?,TRUE, +they saw propaganda,It's true that propaganda inspired some young boys to join the army. Now add another detail from the text. What kind of propaganda was used to convince young boys to join the army?,TRUE, +the propaganda showed the British army needed help in war.,It's true that propaganda was used to inspire some young boys to join the army. Now add another detail from the text. What kind of propaganda was used to convince young boys to join the army?,TRUE, +they believed in the propaganda of war.,It's true that propaganda inspired some young boys to join the army. Now add another detail from the text. What kind of propaganda was used to convince young boys to join the army?,TRUE, +they accepted the propaganda of the war.,It's true that propaganda inspired some young boys to join the army. Now add another detail from the text. What kind of propaganda was used to convince young boys to join the army?,TRUE, +the army didn't have enough volunteers.,It's true that the British Army needed more soldiers during WWI—that's helpful historical context. Write about a specific type of propaganda that convinced some young boys to join the army. Why did some boys feel inspired to join the British Army?,TRUE, +they were surrounded by propaganda.,It's true that propaganda inspired some young boys to join the army. Now add another detail from the text. What kind of propaganda was used to convince young boys to join the army?,TRUE, +they believed in the statements of the military.,It's true that propaganda inspired some young boys to join the army. Now add another detail from the text. What kind of propaganda was used to convince young boys to join the army?,TRUE, +Britain needed more soldiers.,It's true that the British Army needed more soldiers during WWI—that's helpful historical context. Write about a specific type of propaganda that convinced some young boys to join the army. Why did some boys feel inspired to join the British Army?,TRUE, +they were subjected to propaganda from a young age.,It's true that propaganda inspired some young boys to join the army. Now add another detail from the text. What kind of propaganda was used to convince young boys to join the army?,TRUE, +they were regularly shown propaganda for a long time.,It's true that propaganda was used to inspire some young boys to join the army. Now add another detail from the text. What kind of propaganda was used to convince young boys to join the army?,TRUE, +a lot of the information was spread.,It's true that information was spread to encourage young boys to join the army. Now add another detail from the text. What kind of information was used to convince young boys to join the army?,TRUE, +the army needed more soldiers.,It's true that the British Army needed more soldiers during WWI—that's helpful historical context. Write about a specific type of propaganda that convinced some young boys to join the army. Why did some boys feel inspired to join the British Army?,TRUE, +the British army needed more recruits.,It's true that the British Army needed more soldiers during WWI—that's helpful historical context. Write about a specific type of propaganda that convinced some young boys to join the army. Why did some boys feel inspired to join the British Army?,TRUE, +a lot of information was spread.,It's true that information was spread to encourage young boys to join the army. Now add another detail from the text. What kind of information was used to convince young boys to join the army?,TRUE, +their minds were poisoned from a young age with propaganda.,It's true that propaganda influenced some young boys to join the army. Now add another detail from the text. What kind of propaganda was used to convince young boys to join the army?,TRUE, +the army did not have enough volunteers.,It's true that the British Army needed more soldiers during WWI—that's helpful historical context. Write about a specific type of propaganda that convinced some young boys to join the army. Why did some boys feel inspired to join the British Army?,TRUE, +a great deal of propaganda was being spread.,It's true that propaganda inspired some young boys to join the army. Now add another detail from the text. What kind of propaganda was used to convince young boys to join the army?,TRUE, +the British Army was in need of more soldiers.,It's true that the British Army needed more soldiers during WWI—that's helpful historical context. Write about a specific type of propaganda that convinced some young boys to join the army. Why did some boys feel inspired to join the British Army?,TRUE, +they were given propaganda for a long time.,It's true that propaganda was used to inspire some young boys to join the army. Now add another detail from the text. What kind of propaganda was used to convince young boys to join the army?,TRUE, +they were influenced by the propaganda that surrounded them.,It's true that propaganda inspired some young boys to join the army. Now add another detail from the text. What kind of propaganda was used to convince young boys to join the army?,TRUE, +the British didn't have enough volunteers.,It's true that the British Army needed more soldiers during WWI—that's helpful historical context. Write about a specific type of propaganda that convinced some young boys to join the army. Why did some boys feel inspired to join the British Army?,TRUE, +they believed in the propagandas of the war.,It's true that propaganda inspired some young boys to join the army. Now add another detail from the text. What kind of propaganda was used to convince young boys to join the army?,TRUE, +they heard about it everywhere they went due to propaganda,It's true that propaganda inspired some young boys to join the army. Now add another detail from the text. What kind of propaganda was used to convince young boys to join the army?,TRUE, +a lot of propaganda was disseminated.,It's true that propaganda was used to inspire some young boys to join the army. Now add another detail from the text. What kind of propaganda was used to convince young boys to join the army?,TRUE, +the British Army did not have enough volunteers.,It's true that the British Army needed more soldiers during WWI—that's helpful historical context. Write about a specific type of propaganda that convinced some young boys to join the army. Why did some boys feel inspired to join the British Army?,TRUE, +they heard about it a lot a lot due to propaganda,It's true that propaganda inspired some young boys to join the army. Now add another detail from the text. What kind of propaganda was used to convince young boys to join the army?,TRUE, +they heard about it and saw information promoting it,It's true that propaganda inspired some young boys to join the army. Now add another detail from the text. What kind of propaganda was used to convince young boys to join the army?,TRUE, +school was only required for kids under the age of twelve,It's true that school was only mandatory until age 12—that's helpful historical context. Write about a reason that some boys wanted to volunteer for the war instead. Why did some boys feel inspired to join the British Army?,TRUE, +school was only necessary for children up to twelve years old,It's true that school was only mandatory until age 12—that's helpful historical context. Write about a reason that some boys wanted to volunteer for the war instead. Why did some boys feel inspired to join the British Army?,TRUE, +school attendance was only mandatory for children aged 12 or younger,It's true that school was only mandatory until age 12—that's helpful historical context. Write about a reason that some boys wanted to volunteer for the war instead. Why did some boys feel inspired to join the British Army?,TRUE, +school is only mandatory for children under the age of twelve,It's true that school was only mandatory until age 12—that's helpful historical context. Write about a reason that some boys wanted to volunteer for the war instead. Why did some boys feel inspired to join the British Army?,TRUE, +only children below the age of twelve were required to attend school.,It's true that school was only mandatory until age 12—that's helpful historical context. Write about a reason that some boys wanted to volunteer for the war instead. Why did some boys feel inspired to join the British Army?,TRUE, +the school was only mandatory for children under the age of 12 in certain areas.,It's true that school was only mandatory until age 12—that's helpful historical context. Write about a reason that some boys wanted to volunteer for the war instead. Why did some boys feel inspired to join the British Army?,TRUE, +childhood was only mandatory for children under 12,It's true that school was only mandatory until age 12—that's helpful historical context. Write about a reason that some boys wanted to volunteer for the war instead. Why did some boys feel inspired to join the British Army?,TRUE, +school was only required for children aged 12 and under,It's true that school was only mandatory until age 12—that's helpful historical context. Write about a reason that some boys wanted to volunteer for the war instead. Why did some boys feel inspired to join the British Army?,TRUE, +the school was only required until children reached twelve years old.,It's true that school was only mandatory until age 12—that's helpful historical context. Write about a reason that some boys wanted to volunteer for the war instead. Why did some boys feel inspired to join the British Army?,TRUE, +they wanted to serve their country.,Some boys did want to serve their country---that's true! Now add more specific information to your response. Why did some boys feel pressure to volunteer for the army?,FALSE,unhelpful +they felt called to serve.,It's true that many boys felt called to serve. Now add another detail from the text to make your response more specific. Why did some teenage boys believe it was their duty to join the British Army?,TRUE, +school was mandatory until kids were twelve years old.,It's true that school was only mandatory until age 12—that's helpful historical context. Write about a reason that some boys wanted to volunteer for the war instead. Why did some boys feel inspired to join the British Army?,TRUE, +school was mandatory for children under the age of twelve,It's true that school was only mandatory until age 12—that's helpful historical context. Write about a reason that some boys wanted to volunteer for the war instead. Why did some boys feel inspired to join the British Army?,TRUE, +they wanted to fight for there country.,It's true that many boys wanted to fight for their country. Now add another detail from the text to make your response more specific. Why did some teenage boys believe it was their duty to join the British Army?,TRUE, +they wanted to protect their country,It's true that many boys wanted to protect their country. Now add another detail from the text to make your response more specific. Why did some teenage boys believe it was their duty to join the British Army?,TRUE, +they had a calling.,It's true that many boys felt called to serve. Now add another detail from the text to make your response more specific. Why did some teenage boys believe it was their duty to join the British Army?,TRUE, +school was required for kids up to age twelve,It's true that school was only mandatory until age 12—that's helpful historical context. Write about a reason that some boys wanted to volunteer for the war instead. Why did some boys feel inspired to join the British Army?,TRUE, +they wanted to help protect their country.,Some boys did want to protect their country---that's true! Now add more specific information to your response. Why did some boys feel pressure to volunteer for the army?,FALSE,unhelpful +they were called to serve.,It's true that many boys felt called to serve. Now add another detail from the text to make your response more specific. Why did some teenage boys believe it was their duty to join the British Army?,TRUE, +a sense of duty prompted them to do something.,It's true that many boys felt a sense of duty. Now add another detail from the text to make your response more specific. Why did some teenage boys believe it was their duty to join the British Army?,TRUE, +going to school was only required until kids were twelve years old.,It's true that school was only mandatory until age 12—that's helpful historical context. Write about a reason that some boys wanted to volunteer for the war instead. Why did some boys feel inspired to join the British Army?,TRUE, +school was only mandatory for children under twelve,It's true that school was only mandatory until age 12—that's helpful historical context. Write about a reason that some boys wanted to volunteer for the war instead. Why did some boys feel inspired to join the British Army?,TRUE, +they wanted to contribute to the war.,It's true that many boys wanted to contribute to the war effort. Now add another detail from the text to make your response more specific. Why did some teenage boys believe it was their duty to join the British Army?,TRUE, +they wanted to show their support for their country.,Some boys did want to show their support for their country. Now add more specific information to your response. Why did some boys feel inspired to join the British Army?,TRUE, +they wanted to help the cause.,It's true that many boys wanted to help the war effort. Now add another detail from the text to make your response more specific. Why did some teenage boys believe it was their duty to join the British Army?,TRUE, +t felt a call to serve.,It's true that many boys felt called to serve. Now add another detail from the text to make your response more specific. Why did some teenage boys believe it was their duty to join the British Army?,TRUE, +the army required all new soldiers to be at least 18 years old.,"Instead of focusing on a barrier (on something that made it difficult for them to join) to joining the British army, write about a reason why young boys wanted to enlist. Why did some boys want to volunteer for the war?",TRUE, +kids had to go to school until they turned twelve.,It's true that school was only mandatory until age 12—that's helpful historical context. Write about a reason that some boys wanted to volunteer for the war instead. Why did some boys feel inspired to join the British Army?,TRUE, +they felt it was a good cause and wanted to support their country.,Some boys did want to support their country. Now add more specific information to your response. Why did some boys feel inspired to join the British Army?,TRUE, +school was only required until children turned twelve years old.,It's true that school was only mandatory until age 12—that's helpful historical context. Write about a reason that some boys wanted to volunteer for the war instead. Why did some boys feel inspired to join the British Army?,TRUE, +school was only mandatory for children under the age of twelve,It's true that school was only mandatory until age 12—that's helpful historical context. Write about a reason that some boys wanted to volunteer for the war instead. Why did some boys feel inspired to join the British Army?,TRUE, +schools were mandatory for children under the age of twelve,It's true that school was only mandatory until age 12—that's helpful historical context. Write about a reason that some boys wanted to volunteer for the war instead. Why did some boys feel inspired to join the British Army?,TRUE, +they felt they needed to help.,It's true that many boys felt called to serve. Now add another detail from the text to make your response more specific. Why did some teenage boys believe it was their duty to join the British Army?,TRUE, +"due to their age and the legal age requirement of 18, they were often unable to enlist officially.","Instead of focusing on a barrier (on something that made it difficult for them to join) to joining the British army, write about a reason why young boys wanted to enlist. Why did some boys want to volunteer for the war?",TRUE, +they were inspired to serve.,It's true that many boys felt called to serve. Now add another detail from the text to make your response more specific. Why did some teenage boys believe it was their duty to join the British Army?,TRUE, +children were only required to attend school until the age of twelve,It's true that school was only mandatory until age 12—that's helpful historical context. Write about a reason that some boys wanted to volunteer for the war instead. Why did some boys feel inspired to join the British Army?,TRUE, +they were hoping to help the war.,It's true that many boys wanted to help the war effort. Now add another detail from the text to make your response more specific. Why did some teenage boys believe it was their duty to join the British Army?,TRUE, +they wanted to support the war.,It's true that many boys wanted to support the war effort. Now add another detail from the text to make your response more specific. Why did some teenage boys believe it was their duty to join the British Army?,TRUE, +they wanted to fight.,It's true that many boys wanted to fight. Now add another detail from the text to make your response more specific. Why did some teenage boys believe it was their duty to join the British Army?,TRUE, +they felt an urge to serve.,It's true that many boys felt called to serve. Now add another detail from the text to make your response more specific. Why did some teenage boys believe it was their duty to join the British Army?,TRUE, +they wanted to aid the British war effort.,It's true that many boys wanted to aid the British war effort. Now add another detail from the text to make your response more specific. Why did some teenage boys believe it was their duty to join the British Army?,TRUE, +the army required new recruits to be at least 18.,"Instead of focusing on a barrier (on something that made it difficult for them to join) to joining the British army, write about a reason why young boys wanted to enlist. Why did some boys want to volunteer for the war?",TRUE, +children were only required to attend school for a certain number of years,It's true that school was only mandatory until age 12—that's helpful historical context. Write about a reason that some boys wanted to volunteer for the war instead. Why did some boys feel inspired to join the British Army?,TRUE, +only men over 18 were allowed to fight.,"Instead of focusing on a barrier (on something that made it difficult for them to join) to joining the British army, write about a reason why young boys wanted to enlist. Why did some boys want to volunteer for the war?",TRUE, +"one in five that enlisted who were found to be underage, were released from service within the first month.","Instead of focusing on a barrier (on something that made it difficult for them to join) to joining the British army, write about a reason why young boys wanted to enlist. Why did some boys want to volunteer for the war?",TRUE, +the school was only necessary until kids reached the age of twelve.,It's true that school was only mandatory until age 12—that's helpful historical context. Write about a reason that some boys wanted to volunteer for the war instead. Why did some boys feel inspired to join the British Army?,TRUE, +the government required recruits to be at least eighteen years old.,"Instead of focusing on a barrier (on something that made it difficult for them to join) to joining the British army, write about a reason why young boys wanted to enlist. Why did some boys want to volunteer for the war?",TRUE, +they wanted to be in the fight.,It's true that many boys wanted to fight in WWI. Now add another detail from the text to make your response more specific. Why did some teenage boys believe it was their duty to join the British Army?,TRUE, +the army required recruits to be at least 18.,"Instead of focusing on a barrier (on something that made it difficult for them to join) to joining the British army, write about a reason why young boys wanted to enlist. Why did some boys want to volunteer for the war?",TRUE, +school was only required for children under the age of twelve,It's true that school was only mandatory until age 12—that's helpful historical context. Write about a reason that some boys wanted to volunteer for the war instead. Why did some boys feel inspired to join the British Army?,TRUE, +1 in 5 of the underage boys were sent home within a month,"Instead of focusing on a barrier (on something that made it difficult for them to join) to joining the British army, write about a reason why young boys wanted to enlist. Why did some boys want to volunteer for the war?",TRUE, +1 in 5 of the underage soldiers were discharged within a month,"Instead of focusing on a barrier (on something that made it difficult for them to join) to joining the British army, write about a reason why young boys wanted to enlist. Why did some boys want to volunteer for the war?",TRUE, +a large number of underage boys were sent back home during their first month,"Instead of focusing on a barrier (on something that made it difficult for them to join) to joining the British army, write about a reason why young boys wanted to enlist. Why did some boys want to volunteer for the war?",TRUE, +"some who looked old enough were able to simply give fake names and ages when they went to volunteer, since many didn't have identification documents like birth certificates.","That's something that happened as a result of young boys' interest in joining the army. Since this sentence uses 'because,' write about a reason that they wanted to enlist instead. Why did some boys want to volunteer for the war?",TRUE, +many underage soldiers were discharged once their parents verified their ages.,"Instead of focusing on a barrier (on something that made it difficult for them to join) to joining the British army, write about a reason why young boys wanted to enlist. Why did some boys want to volunteer for the war?",TRUE, +some joined the army using fake IDs and names.,"That's something that happened as a result of young boys' interest in joining the army. Since this sentence uses 'because,' write about a reason that they wanted to enlist instead. Why did some boys want to volunteer for the war?",TRUE, +many underage soldiers were sent home once their parents verified their ages,"Instead of focusing on a barrier (on something that made it difficult for them to join) to joining the British army, write about a reason why young boys wanted to enlist. Why did some boys want to volunteer for the war?",TRUE, +they used fake names and ages to get around age requirement.,"That's something that happened as a result of young boys' interest in joining the army. Since this sentence uses 'because,' write about a reason that they wanted to enlist instead. Why did some boys want to volunteer for the war?",TRUE, +"once they were on the battlefield, their inexperience and immaturity forced them to return to their homes, to parents who reported them missing.","Instead of focusing on what happened after they joined the army, write about why some young boys wanted to join the army in the first place. Check the text for reasons why they volunteered.",TRUE, +1 out of 5 underage boys was sent home within a month,"Instead of focusing on a barrier (on something that made it difficult for them to join) to joining the British army, write about a reason why young boys wanted to enlist. Why did some boys want to volunteer for the war?",TRUE, +many underage soldiers were dismissed once their parents could prove their age.,"Instead of focusing on a barrier (on something that made it difficult for them to join) to joining the British army, write about a reason why young boys wanted to enlist. Why did some boys want to volunteer for the war?",TRUE, +many underage soldiers were sent home once their parents verified their age.,"Instead of focusing on a barrier (on something that made it difficult for them to join) to joining the British army, write about a reason why young boys wanted to enlist. Why did some boys want to volunteer for the war?",TRUE, +many underage soldiers were sent home once their parents confirmed their ages.,"Instead of focusing on a barrier (on something that made it difficult for them to join) to joining the British army, write about a reason why young boys wanted to enlist. Why did some boys want to volunteer for the war?",TRUE, +many underage soldiers were sent home once their parents verified their ages.,"Instead of focusing on a barrier (on something that made it difficult for them to join) to joining the British army, write about a reason why young boys wanted to enlist. Why did some boys want to volunteer for the war?",TRUE, +a sizable number of boys provided false identities and ages in order to join.,"That's something that happened as a result of young boys' interest in joining the army. Since this sentence uses 'because,' write about a reason that they wanted to enlist instead. Why did some boys want to volunteer for the war?",TRUE, +one in five underage boys aresent home within a month.,"Instead of focusing on a barrier (on something that made it difficult for them to join) to joining the British army, write about a reason why young boys wanted to enlist. Why did some boys want to volunteer for the war?",TRUE, +many boys gave fake names and ages to be able to join.,"That's something that happened as a result of young boys' interest in joining the army. Since this sentence uses 'because,' write about a reason that they wanted to enlist instead. Why did some boys want to volunteer for the war?",TRUE, +many boys gave fake names and ages to be able to participate.,"That's something that happened as a result of young boys' interest in joining the army. Since this sentence uses 'because,' write about a reason that they wanted to enlist instead. Why did some boys want to volunteer for the war?",TRUE, +they gave fake names and ages to enlist.,"That's something that happened as a result of young boys' interest in joining the army. Since this sentence uses 'because,' write about a reason that they wanted to enlist instead. Why did some boys want to volunteer for the war?",TRUE, +many of them lied about their age to get in the service.,"That's something that happened as a result of young boys' interest in joining the army. Since this sentence uses 'because,' write about a reason that they wanted to enlist instead. Why did some boys want to volunteer for the war?",TRUE, +the ones who were underage faked their names and ages to get in.,"That's something that happened as a result of young boys' interest in joining the army. Since this sentence uses 'because,' write about a reason that they wanted to enlist instead. Why did some boys want to volunteer for the war?",TRUE, +many underage soldiers were sent home once their parents provided verification of their age.,"Instead of focusing on a barrier (on something that made it difficult for them to join) to joining the British army, write about a reason why young boys wanted to enlist. Why did some boys want to volunteer for the war?",TRUE, +1 out of 5 underage boys were sent home within a month,"Instead of focusing on a barrier (on something that made it difficult for them to join) to joining the British army, write about a reason why young boys wanted to enlist. Why did some boys want to volunteer for the war?",TRUE, +many boys gave fake names and ages in order to be able to join.,"That's something that happened as a result of young boys' interest in joining the army. Since this sentence uses 'because,' write about a reason that they wanted to enlist instead. Why did some boys want to volunteer for the war?",TRUE, +a high percentage of underage soldiers were discharged once their parents verified their age.,"Instead of focusing on a barrier (on something that made it difficult for them to join) to joining the British army, write about a reason why young boys wanted to enlist. Why did some boys want to volunteer for the war?",TRUE, +many boys gave fake names and ages to join a group.,"That's something that happened as a result of young boys' interest in joining the army. Since this sentence uses 'because,' write about a reason that they wanted to enlist instead. Why did some boys want to volunteer for the war?",TRUE, +many of them lied about their age to get in the military service.,"That's something that happened as a result of young boys' interest in joining the army. Since this sentence uses 'because,' write about a reason that they wanted to enlist instead. Why did some boys want to volunteer for the war?",TRUE, +many underage soldiers were sent home once their parents verified their age,"Instead of focusing on a barrier (on something that made it difficult for them to join) to joining the British army, write about a reason why young boys wanted to enlist. Why did some boys want to volunteer for the war?",TRUE, +some of them provided fake names and lied about their real ages in order to enlist.,"That's something that happened as a result of young boys' interest in joining the army. Since this sentence uses 'because,' write about a reason that they wanted to enlist instead. Why did some boys want to volunteer for the war?",TRUE, +they gave fake names and ages when they went to volunteer.,"That's something that happened as a result of young boys' interest in joining the army. Since this sentence uses 'because,' write about a reason that they wanted to enlist instead. Why did some boys want to volunteer for the war?",TRUE, +many boys gave false names and ages to be able to join.,"That's something that happened as a result of young boys' interest in joining the army. Since this sentence uses 'because,' write about a reason that they wanted to enlist instead. Why did some boys want to volunteer for the war?",TRUE, +many boys falsified their identities and ages in order to join.,"That's something that happened as a result of young boys' interest in joining the army. Since this sentence uses 'because,' write about a reason that they wanted to enlist instead. Why did some boys want to volunteer for the war?",TRUE, +"one in five who enlisted were found to be underage, and were released from service within the first month.","Instead of focusing on a barrier (on something that made it difficult for them to join) to joining the British army, write about a reason why young boys wanted to enlist. Why did some boys want to volunteer for the war?",TRUE, +many of them lied about their age to get into the service.,"That's something that happened as a result of young boys' interest in joining the army. Since this sentence uses 'because,' write about a reason that they wanted to enlist instead. Why did some boys want to volunteer for the war?",TRUE, +many lied about their age to get in the service.,"That's something that happened as a result of young boys' interest in joining the army. Since this sentence uses 'because,' write about a reason that they wanted to enlist instead. Why did some boys want to volunteer for the war?",TRUE, +"these boys found various other ways to join the British Army, such as by using fake identities.","That's something that happened as a result of young boys' interest in joining the army. Since this sentence uses 'because,' write about a reason that they wanted to enlist instead. Why did some boys want to volunteer for the war?",TRUE, +the boys wanted to find their own pathss.,Try clearing your response and starting again. Why did some boys want to volunteer for the war? Check that your response only uses information from the text.,TRUE, +they wanted girls to look at them.,Try clearing your response and starting again. Why did some boys want to volunteer for the war? Check that your response only uses information from the text.,TRUE, +"these boys found various other ways to enlist, such as using fake identities.","That's something that happened as a result of young boys' interest in joining the army. Since this sentence uses 'because,' write about a reason that they wanted to enlist instead. Why did some boys want to volunteer for the war?",TRUE, +they felt like it would be a fun adventure,Try clearing your response and starting again. Why did some boys want to volunteer for the war? Check that your response only uses information from the text.,TRUE, +the boys wanted to carve out their own pathways.,Try clearing your response and starting again. Why did some boys want to volunteer for the war? Check that your response only uses information from the text.,TRUE, +they would get money out of it.,Try clearing your response and starting again. Why did some boys want to volunteer for the war? Check that your response only uses information from the text.,TRUE, +they thought they were grown up,Try clearing your response and starting again. Why did some boys want to volunteer for the war? Check that your response only uses information from the text.,TRUE, +the boys desired to chart their own trajectories.,Try clearing your response and starting again. Why did some boys want to volunteer for the war? Check that your response only uses information from the text.,TRUE, +they thought it would be fun,Try clearing your response and starting again. Why did some boys want to volunteer for the war? Check that your response only uses information from the text.,TRUE, +they needed a regular paycheck.,Try clearing your response and starting again. Why did some boys want to volunteer for the war? Check that your response only uses information from the text.,TRUE, +it supports Britai's economy.,Try clearing your response and starting again. Why did some boys want to volunteer for the war? Check that your response only uses information from the text.,TRUE, +the boys wanted to chart their own courses.,Try clearing your response and starting again. Why did some boys want to volunteer for the war? Check that your response only uses information from the text.,TRUE, +they wanted to see what war is like,Try clearing your response and starting again. Why did some boys want to volunteer for the war? Check that your response only uses information from the text.,TRUE, +the boys were pressured to join,Some boys did feel pressure to enlist in the British Army---that's true! Now add more specific information to your response. Why did some boys feel pressure to volunteer for the army?,TRUE, +they wanted to help,It's true that many boys wanted to help their country. Now add another detail from the text to make your response more specific. Why did some teenage boys believe it was their duty to join the British Army?,TRUE, +"these boys found various other ways to enlist, such as by falsifying their identities.","That's something that happened as a result of young boys' interest in joining the army. Since this sentence uses 'because,' write about a reason that they wanted to enlist instead. Why did some boys want to volunteer for the war?",TRUE, +they needed a regular salary.,Try clearing your response and starting again. Why did some boys want to volunteer for the war? Check that your response only uses information from the text.,TRUE, +that it was there job.,It's true that many boys felt it was their duty to join the war effort. Now add another detail from the text to make your response more specific. Why did some teenage boys believe it was their duty to join the British Army?,TRUE, +they wanted to carve out their own identities.,Try clearing your response and starting again. Why did some boys want to volunteer for the war? Check that your response only uses information from the text.,TRUE, +some wanted to do the job and help the community.,Try clearing your response and starting again. Why did some boys want to volunteer for the war? Check that your response only uses information from the text.,TRUE, +there is a high chance that they will lose.,"It's true that there was a high chance of death in WWI. However, this sentence asks for the reasons why young boys *wanted* to volunteer. Please review the text and explain why some boys were motivated to join the British Army despite the risks.",FALSE,bonkers +they needed money,Try clearing your response and starting again. Why did some boys want to volunteer for the war? Check that your response only uses information from the text.,TRUE, +there was a large chance of them losing.,It's true that there was a large chance of them losing. Now add another detail from the text to make your response more specific. Why did some teenage boys believe it was their duty to join the British Army?,FALSE,bonkers +some wanted to save the people they love.,It's true that many boys had loved ones they wanted to protect. Now add another detail from the text to make your response more specific. Why did some teenage boys believe it was their duty to join the British Army?,FALSE,bonkers +there was a large chances of them losing.,"It's true that there was a chance of losing the war. However, this response does not explain why many young teenage boys wanted to volunteer in the British Army during WWI. Try to use information from the text to explain why they volunteered.",FALSE,bonkers +Great Britain needed men to fight.,It's true that the British Army needed more soldiers during WWI—that's helpful historical context. Write about a specific type of propaganda that convinced some young boys to join the army. Why did some boys feel inspired to join the British Army?,TRUE, +the military needed more soldiers.,It's true that the British Army needed more soldiers during WWI—that's helpful historical context. Write about a specific type of propaganda that convinced some young boys to join the army. Why did some boys feel inspired to join the British Army?,TRUE, +some beliefs about belonging superiority both inspired less powerful groups to fight against that expansion as well.,"It's true that nationalist beliefs inspired some to fight. However, this response doesn't explain why *young teenage boys* specifically wanted to volunteer. Please reread the text and focus on the reasons given for why young boys wanted to join the British Army during WWI.",FALSE,bonkers +they were swayed by the propaganda around them.,It's true that propaganda inspired some young boys to join the army. Now add another detail from the text. What kind of propaganda was used to convince young boys to join the army?,TRUE, +Great Britain was going to war and the Army need more men to join.,It's true that the British Army needed more soldiers during WWI—that's helpful historical context. Write about a specific type of propaganda that convinced some young boys to join the army. Why did some boys feel inspired to join the British Army?,TRUE, +"these boys found various other ways to enlist, including using fake identities.","That's something that happened as a result of young boys' interest in joining the army. Since this sentence uses 'because,' write about a reason that they wanted to enlist instead. Why did some boys want to volunteer for the war?",TRUE, +it supports Britain's economy.,"It's true that Britain's economy was impacted by WWI. However, this response does not explain why many young teenage boys *wanted* to volunteer for the British Army. The text discusses reasons why boys felt motivated to join; please review the text and try again.",FALSE,bonkers +they were influenced by the propaganda around them.,It's true that propaganda inspired some young boys to join the army. Now add another detail from the text. What kind of propaganda was used to convince young boys to join the army?,TRUE, +the British Army needed more soldiers to fulfill its obligations.,It's true that the British Army needed more soldiers during WWI—that's helpful historical context. Write about a specific type of propaganda that convinced some young boys to join the army. Why did some boys feel inspired to join the British Army?,TRUE, +they were influenced by the propaganda of the surrounding environment.,It's true that propaganda inspired some young boys to join the army. Now add another detail from the text. What kind of propaganda was used to convince young boys to join the army?,TRUE, +the school was only mandatory for children under the age of 12.,It's true that school was only mandatory until age 12—that's helpful historical context. Write about a reason that some boys wanted to volunteer for the war instead. Why did some boys feel inspired to join the British Army?,TRUE, +school was only required for kids under the age of 12,It's true that school was only mandatory until age 12—that's helpful historical context. Write about a reason that some boys wanted to volunteer for the war instead. Why did some boys feel inspired to join the British Army?,TRUE, +British boys had pressure to serve,Some boys did feel pressure to enlist in the British Army---that's true! Now add more specific information to your response. Why did some boys feel pressure to volunteer for the army?,TRUE, \ No newline at end of file diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/data_fetcher.rb b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/data_fetcher.rb index 3ffbfa8d337..e06606cf701 100644 --- a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/data_fetcher.rb +++ b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/data_fetcher.rb @@ -6,9 +6,12 @@ module Evaluation class DataFetcher < ApplicationService Item = Data.define(:entry, :feedback, :optimal, :rule) - WHALES = 'whales_so.csv' - SOLDIERS = 'underage_soldiers_but.csv' - DEFAULT_FILE = WHALES + WHALES_SO = 'whales_so.csv' + SOLDIERS_BUT = 'underage_soldiers_but.csv' + SOLDIERS_BECAUSE = 'underage_soldiers_because.csv' + HAITIAN_BECAUSE = 'new_haitian_because.csv' + + DEFAULT_FILE = HAITIAN_BECAUSE CSV_FILE_PATH = "#{Evidence::Engine.root}/app/services/evidence/gen_ai/evaluation/data/%s" attr_reader :rule, :optimal diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/feedback_evaluator.rb b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/feedback_evaluator.rb index 597de1e88d4..9a93dabe795 100644 --- a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/feedback_evaluator.rb +++ b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/feedback_evaluator.rb @@ -7,7 +7,6 @@ class FeedbackEvaluator < ApplicationService Result = Data.define(:optimal, :errors) CHECKS = [ - Misacknowledgement::Checker, Tone::Checker, Grammar::Checker, Verbose::Checker, diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/tone/checker.rb b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/tone/checker.rb index 8c433ef0088..854b3f19f25 100644 --- a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/tone/checker.rb +++ b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/tone/checker.rb @@ -5,7 +5,7 @@ module GenAI module Evaluation module Tone class Checker < Evaluation::BaseChecker - TONE_REGEX = /(That\'s incorrect|To make your response better|unrelated|off-topic|Can you|You\'re right|You didn\'t|Please)/ + TONE_REGEX = /(That\'s incorrect|To make your response better|unrelated|off-topic|Can you|You\'re right|You didn\'t|Please|sentence starts with)/ def run return suboptimal if feedback.match?(TONE_REGEX) From 804a6ec66f90d86f9ef514a338d0aee5ac40ecdd Mon Sep 17 00:00:00 2001 From: Daniel Drabik Date: Wed, 22 Jan 2025 17:01:37 -0500 Subject: [PATCH 18/51] Spec wip. --- .../evidence/app/services/evidence/scalpel.rb | 12 ++- .../lib/core_ext/string_extensions.rb | 4 - .../spec/services/evidence/scalpel_spec.rb | 95 +++++++++++++++++++ 3 files changed, 102 insertions(+), 9 deletions(-) create mode 100644 services/QuillLMS/engines/evidence/spec/services/evidence/scalpel_spec.rb diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/scalpel.rb b/services/QuillLMS/engines/evidence/app/services/evidence/scalpel.rb index 33a0eceed80..1bc5ce0edbb 100644 --- a/services/QuillLMS/engines/evidence/app/services/evidence/scalpel.rb +++ b/services/QuillLMS/engines/evidence/app/services/evidence/scalpel.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true # -# File copied from https://github.com/louismullie/scalpel since RubyGems isn't up to date (and it's only one file) +# File copied (and edited) from a 1 file gem at https://github.com/louismullie/scalpel +# since RubyGems doesn't have the last version and there were some bugs / needed better test coverage # # Sentence segmentation based on a set of predefined # rules that handle a large number of usage cases of @@ -17,9 +18,9 @@ class Evidence::Scalpel def self.cut(text) # Get a copy of the string. - text = text.to_s + text = text.dup.to_s # Remove composite abbreviations. - text.gsub!('et al.', '&&&') + text.gsub!('et al.', 'etal.') # Remove suspension points. text.gsub!('...', '&;&.') # Remove floating point numbers. @@ -43,10 +44,11 @@ def self.cut(text) text.gsub!(/([.?!])\s?"/) { '&$&' + $1 } # Remove sentence enders before parens. text.gsub!(/([.?!])\s?\)/) { '&€&' + $1 } + # Split on any sentence ender. sentences = text.split(/([.!?])/) new_sents = [] - # Join the obtaine slices. + # Join the obtained slices. sentences.each_slice(2) do |slice| new_sents << slice.join('') end @@ -56,7 +58,7 @@ def self.cut(text) # Skip whitespace zones. next if sentence.strip == '' # Repair composite abbreviations. - sentence.gsub!('&&&', 'et al.') + sentence.gsub!('etal', 'et al') # Repair initials. sentence.gsub!("&£&", ".") # Repair abbreviations. diff --git a/services/QuillLMS/engines/evidence/lib/core_ext/string_extensions.rb b/services/QuillLMS/engines/evidence/lib/core_ext/string_extensions.rb index 215b5e385cd..200e870e823 100644 --- a/services/QuillLMS/engines/evidence/lib/core_ext/string_extensions.rb +++ b/services/QuillLMS/engines/evidence/lib/core_ext/string_extensions.rb @@ -23,8 +23,4 @@ def question_count count + questions_in_sentence end end - - def string_before(delimiter) - include?(delimiter) ? split(delimiter, 2).first : self - end end diff --git a/services/QuillLMS/engines/evidence/spec/services/evidence/scalpel_spec.rb b/services/QuillLMS/engines/evidence/spec/services/evidence/scalpel_spec.rb new file mode 100644 index 00000000000..d6e402e7dc6 --- /dev/null +++ b/services/QuillLMS/engines/evidence/spec/services/evidence/scalpel_spec.rb @@ -0,0 +1,95 @@ +# frozen_string_literal: true + +require 'rails_helper' + +RSpec.describe Evidence::Scalpel do + Example = Data.define(:type, :text, :result) + + EXAMPLES = [ + Example.new( + type: 'simple sentences', + text: 'This is a sentence. This is another one!', + result: ["This is a sentence.", "This is another one!"] + ), + Example.new( + type: 'sentence ending abbreviation', + text: 'Dr. Smith went to Washington, etc. It was sunny.', + result: ["Dr. Smith went to Washington, etc.", "It was sunny."] + ), + Example.new( + type: 'sentence ending abbreviation with et al.', + text: 'Dr. Smith went to Washington, et al. It was sunny.', + result: ["Dr. Smith went to Washington, et al.", "It was sunny."] + ), + Example.new( + type: 'abbreviations', + text: 'Dr. Smith went to Washington. It was sunny.', + result: ["Dr. Smith went to Washington.", "It was sunny."] + ), + Example.new( + type: 'e.g.', + text: "Please bring the necessary supplies, e.g. pens, paper, and books.", + result: ["Please bring the necessary supplies, e.g. pens, paper, and books."] + ), + Example.new( + type: 'etc.', + text: "Please bring the necessary supplies like pens, paper, etc., and don't forget your notebook.", + result: ["Please bring the necessary supplies like pens, paper, etc., and don't forget your notebook."] + ), + Example.new( + type: 'numbers with periods', + text: 'The value is 3.14. That is pi.', + result: ["The value is 3.14.", "That is pi."] + ), + Example.new( + type: 'ellipses', + text: 'I was thinking... maybe we should go. What do you think?', + result: ["I was thinking...", "maybe we should go.", "What do you think?"] + ), + Example.new( + type: 'titles and initials', + text: 'Mr. J.D. Rockefeller was a famous figure. He lived in the U.S.', + result: ["Mr. J.D. Rockefeller was a famous figure.", "He lived in the U.S."] + ), + Example.new( + type: 'nested quotes', + text: "'Did he say, \"Yes\"?' she asked.", + result: ["'Did he say, \"Yes\"?'", "she asked."] + ), + Example.new( + type: 'punctuation near quotes', + text: "She said, 'Hello.' Then she left.", + result: ["She said, 'Hello.'", "Then she left."] + ), + # Currently failing, this is a bug + # Example.new( + # type: 'parentheses with sentence enders', + # text: 'He was late (again!). She was upset.', + # result: ["He was late (again!).", "She was upset."] + # ), + Example.new( + type: 'extra whitespace', + text: ' Hello world. This is a test. ', + result: ["Hello world.", "This is a test."] + ), + Example.new( + type: 'empty input', + text: '', + result: [] + ), + Example.new( + type: 'floating point numbers without leading zeros', + text: 'The value was .5. It increased to 1.5.', + result: ["The value was .5.", "It increased to 1.5."] + ) + ] + describe '.cut' do + EXAMPLES.each do |example| + context example.type do + subject { described_class.cut(example.text) } + + it { expect(subject).to eq(example.result) } + end + end + end +end From 9959c1f3c026dbca44969825fa800190c75ece9a Mon Sep 17 00:00:00 2001 From: Daniel Drabik Date: Thu, 23 Jan 2025 10:02:52 -0500 Subject: [PATCH 19/51] Fix specs, update Scalpel code for more edge cases. --- .../evidence/app/services/evidence/scalpel.rb | 15 +- .../spec/services/evidence/scalpel_spec.rb | 191 ++++++++++-------- 2 files changed, 118 insertions(+), 88 deletions(-) diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/scalpel.rb b/services/QuillLMS/engines/evidence/app/services/evidence/scalpel.rb index 1bc5ce0edbb..727b733592d 100644 --- a/services/QuillLMS/engines/evidence/app/services/evidence/scalpel.rb +++ b/services/QuillLMS/engines/evidence/app/services/evidence/scalpel.rb @@ -11,7 +11,7 @@ class Evidence::Scalpel # Current version. - VERSION = '0.2.2' + VERSION = '0.2.3' # Segment a text using the Scalpel algorithm. # This will eventually be ported to a gem. @@ -21,6 +21,8 @@ def self.cut(text) text = text.dup.to_s # Remove composite abbreviations. text.gsub!('et al.', 'etal.') + # Remove list enders before a comma. + text.gsub!(/(etc|etal)(\.)(\,)/) { $1 + '&|&' + $3} # Remove suspension points. text.gsub!('...', '&;&.') # Remove floating point numbers. @@ -43,7 +45,9 @@ def self.cut(text) text.gsub!(/([.?!])\s?'/) { '&%&' + $1 } text.gsub!(/([.?!])\s?"/) { '&$&' + $1 } # Remove sentence enders before parens. - text.gsub!(/([.?!])\s?\)/) { '&€&' + $1 } + text.gsub!(/([.])\s?(\))/) { '&€1&' + $2 } + text.gsub!(/([?])\s?(\))/) { '&€2&' + $2 } + text.gsub!(/([!])\s?(\))/) { '&€3&' + $2 } # Split on any sentence ender. sentences = text.split(/([.!?])/) @@ -59,6 +63,8 @@ def self.cut(text) next if sentence.strip == '' # Repair composite abbreviations. sentence.gsub!('etal', 'et al') + # Repair list enders before a comma. + sentence.gsub!("&\|&", ".") # Repair initials. sentence.gsub!("&£&", ".") # Repair abbreviations. @@ -77,7 +83,10 @@ def self.cut(text) sentence.gsub!(/&\*&([.?!])/) { "'" + $1 + '”' } sentence.gsub!(/&\$&([.!?])/) { $1 + '"' } # Repair parens with sentence enders - sentence.gsub!(/&€&([.!?])/) { $1 + ')' } + sentence.gsub!("&€1&", ".") + sentence.gsub!("&€2&", "?") + sentence.gsub!("&€3&", "!") + # Repair floats without leading zeros. sentence.gsub!(/&#&([0-9]+)/) { '.' + $1 } results << sentence.strip diff --git a/services/QuillLMS/engines/evidence/spec/services/evidence/scalpel_spec.rb b/services/QuillLMS/engines/evidence/spec/services/evidence/scalpel_spec.rb index d6e402e7dc6..e4bb8f94ae8 100644 --- a/services/QuillLMS/engines/evidence/spec/services/evidence/scalpel_spec.rb +++ b/services/QuillLMS/engines/evidence/spec/services/evidence/scalpel_spec.rb @@ -3,93 +3,114 @@ require 'rails_helper' RSpec.describe Evidence::Scalpel do - Example = Data.define(:type, :text, :result) - - EXAMPLES = [ - Example.new( - type: 'simple sentences', - text: 'This is a sentence. This is another one!', - result: ["This is a sentence.", "This is another one!"] - ), - Example.new( - type: 'sentence ending abbreviation', - text: 'Dr. Smith went to Washington, etc. It was sunny.', - result: ["Dr. Smith went to Washington, etc.", "It was sunny."] - ), - Example.new( - type: 'sentence ending abbreviation with et al.', - text: 'Dr. Smith went to Washington, et al. It was sunny.', - result: ["Dr. Smith went to Washington, et al.", "It was sunny."] - ), - Example.new( - type: 'abbreviations', - text: 'Dr. Smith went to Washington. It was sunny.', - result: ["Dr. Smith went to Washington.", "It was sunny."] - ), - Example.new( - type: 'e.g.', - text: "Please bring the necessary supplies, e.g. pens, paper, and books.", - result: ["Please bring the necessary supplies, e.g. pens, paper, and books."] - ), - Example.new( - type: 'etc.', - text: "Please bring the necessary supplies like pens, paper, etc., and don't forget your notebook.", - result: ["Please bring the necessary supplies like pens, paper, etc., and don't forget your notebook."] - ), - Example.new( - type: 'numbers with periods', - text: 'The value is 3.14. That is pi.', - result: ["The value is 3.14.", "That is pi."] - ), - Example.new( - type: 'ellipses', - text: 'I was thinking... maybe we should go. What do you think?', - result: ["I was thinking...", "maybe we should go.", "What do you think?"] - ), - Example.new( - type: 'titles and initials', - text: 'Mr. J.D. Rockefeller was a famous figure. He lived in the U.S.', - result: ["Mr. J.D. Rockefeller was a famous figure.", "He lived in the U.S."] - ), - Example.new( - type: 'nested quotes', - text: "'Did he say, \"Yes\"?' she asked.", - result: ["'Did he say, \"Yes\"?'", "she asked."] - ), - Example.new( - type: 'punctuation near quotes', - text: "She said, 'Hello.' Then she left.", - result: ["She said, 'Hello.'", "Then she left."] - ), - # Currently failing, this is a bug - # Example.new( - # type: 'parentheses with sentence enders', - # text: 'He was late (again!). She was upset.', - # result: ["He was late (again!).", "She was upset."] - # ), - Example.new( - type: 'extra whitespace', - text: ' Hello world. This is a test. ', - result: ["Hello world.", "This is a test."] - ), - Example.new( - type: 'empty input', - text: '', - result: [] - ), - Example.new( - type: 'floating point numbers without leading zeros', - text: 'The value was .5. It increased to 1.5.', - result: ["The value was .5.", "It increased to 1.5."] - ) - ] describe '.cut' do - EXAMPLES.each do |example| - context example.type do - subject { described_class.cut(example.text) } + it 'handles simple sentences correctly' do + text = 'This is a sentence. This is another one!' + result = ["This is a sentence.", "This is another one!"] + expect(described_class.cut(text)).to eq(result) + end + + it 'handles sentence ending abbreviation' do + text = 'Dr. Smith went to Washington, etc. It was sunny.' + result = ["Dr. Smith went to Washington, etc.", "It was sunny."] + expect(described_class.cut(text)).to eq(result) + end + + it 'handles sentence ending abbreviation with et al.' do + text = 'Dr. Smith went to Washington, et al. It was sunny.' + result = ["Dr. Smith went to Washington, et al.", "It was sunny."] + expect(described_class.cut(text)).to eq(result) + end + + it 'handles abbreviations correctly' do + text = 'Dr. Smith went to Washington. It was sunny.' + result = ["Dr. Smith went to Washington.", "It was sunny."] + expect(described_class.cut(text)).to eq(result) + end + + it 'handles e.g. correctly' do + text = "Please bring the necessary supplies, e.g. pens, paper, and books." + result = ["Please bring the necessary supplies, e.g. pens, paper, and books."] + expect(described_class.cut(text)).to eq(result) + end + + it 'handles etc. correctly' do + text = "Please bring the necessary supplies like pens, paper, etc., and don't forget your notebook." + result = ["Please bring the necessary supplies like pens, paper, etc., and don't forget your notebook."] + expect(described_class.cut(text)).to eq(result) + end + + # Failing test. Might need LLM if we need this. + # it 'common test sentence, mid-abbreviations followed by capital and sentence-ending abbreviation followed by capital' do + # text = "Hello world. My name is Mr. Smith. I work for the U.S. Government and I live in the U.S. I live in New York." + # result = ["Hello world.", "My name is Mr. Smith.", "I work for the U.S. Government and I live in the U.S.", "I live in New York."] + # expect(described_class.cut(text)).to eq(result) + # end + + it 'handles etc. as a sentence ender' do + text = "Please bring the necessary supplies like pens, paper, etc. Also, don't forget your notebook." + result = ["Please bring the necessary supplies like pens, paper, etc.", "Also, don't forget your notebook."] + expect(described_class.cut(text)).to eq(result) + end + + it 'handles numbers with periods' do + text = 'The value is 3.14. That is pi.' + result = ["The value is 3.14.", "That is pi."] + expect(described_class.cut(text)).to eq(result) + end + + it 'handles ellipses correctly' do + text = 'I was thinking... maybe we should go. What do you think?' + result = ["I was thinking...", "maybe we should go.", "What do you think?"] + expect(described_class.cut(text)).to eq(result) + end + + it 'handles titles and initials correctly' do + text = 'Mr. J.D. Rockefeller was a famous figure. He lived in the U.S.' + result = ["Mr. J.D. Rockefeller was a famous figure.", "He lived in the U.S."] + expect(described_class.cut(text)).to eq(result) + end + + it 'handles nested quotes correctly' do + text = "'Did he say, \"Yes\"?' she asked." + result = ["'Did he say, \"Yes\"?'", "she asked."] + expect(described_class.cut(text)).to eq(result) + end + + it 'handles punctuation near quotes correctly' do + text = "She said, 'Hello.' Then she left." + result = ["She said, 'Hello.'", "Then she left."] + expect(described_class.cut(text)).to eq(result) + end + + it 'handles parentheses with sentence enders correctly' do + text = 'He was late (again!). She was upset.' + result = ["He was late (again!).", "She was upset."] + expect(described_class.cut(text)).to eq(result) + end + + it 'handles parentheses within sentence if it has sentence ender' do + text = "He was latte (spelling?), but that isn't important. She was upset." + result = ["He was latte (spelling?), but that isn't important.", "She was upset."] + expect(described_class.cut(text)).to eq(result) + end + + it 'handles extra whitespace correctly' do + text = ' Hello world. This is a test. ' + result = ["Hello world.", "This is a test."] + expect(described_class.cut(text)).to eq(result) + end + + it 'returns an empty array for empty input' do + text = '' + result = [] + expect(described_class.cut(text)).to eq(result) + end - it { expect(subject).to eq(example.result) } - end + it 'handles floating point numbers without leading zeros' do + text = 'The value was .5. It increased to 1.5.' + result = ["The value was .5.", "It increased to 1.5."] + expect(described_class.cut(text)).to eq(result) end end end From 64d12a190f9130f07aa1a71eba1467c707d45d5c Mon Sep 17 00:00:00 2001 From: Daniel Drabik Date: Thu, 23 Jan 2025 10:58:50 -0500 Subject: [PATCH 20/51] Add some basic tests. --- .../lib/core_ext/string_extensions_spec.rb | 61 ++++++++++++++++++ .../direction_count/checker_spec.rb | 37 +++++++++++ .../evaluation/feedback_evaluator_spec.rb | 60 +++++++++++++++++ .../gen_ai/evaluation/grammar/checker_spec.rb | 46 +++++++++++++ .../gen_ai/evaluation/tone/checker_spec.rb | 64 +++++++++++++++++++ .../gen_ai/evaluation/verbose/checker_spec.rb | 55 ++++++++++++++++ 6 files changed, 323 insertions(+) create mode 100644 services/QuillLMS/engines/evidence/spec/lib/core_ext/string_extensions_spec.rb create mode 100644 services/QuillLMS/engines/evidence/spec/services/evidence/gen_ai/evaluation/direction_count/checker_spec.rb create mode 100644 services/QuillLMS/engines/evidence/spec/services/evidence/gen_ai/evaluation/feedback_evaluator_spec.rb create mode 100644 services/QuillLMS/engines/evidence/spec/services/evidence/gen_ai/evaluation/grammar/checker_spec.rb create mode 100644 services/QuillLMS/engines/evidence/spec/services/evidence/gen_ai/evaluation/tone/checker_spec.rb create mode 100644 services/QuillLMS/engines/evidence/spec/services/evidence/gen_ai/evaluation/verbose/checker_spec.rb diff --git a/services/QuillLMS/engines/evidence/spec/lib/core_ext/string_extensions_spec.rb b/services/QuillLMS/engines/evidence/spec/lib/core_ext/string_extensions_spec.rb new file mode 100644 index 00000000000..2048b8435a3 --- /dev/null +++ b/services/QuillLMS/engines/evidence/spec/lib/core_ext/string_extensions_spec.rb @@ -0,0 +1,61 @@ +require 'rails_helper' + +describe "String#sentences and related methods" do + let(:input_string) { "Who are you? What do you want?" } + let(:stubbed_sentences) { ["Who are you?", "What do you want?"] } + + before do + allow(Evidence::Scalpel).to receive(:cut).and_return(stubbed_sentences) + end + + subject { input_string } + + describe "#sentences" do + it "returns an array of sentences using the Scalpel.cut method" do + expect(subject.sentences).to eq(stubbed_sentences) + end + + it "calls Scalpel.cut with the input string" do + subject.sentences + expect(Evidence::Scalpel).to have_received(:cut).with(input_string) + end + end + + describe "#sentence_count" do + it "returns the number of sentences" do + expect(subject.sentence_count).to eq(stubbed_sentences.size) + end + end + + describe "#question_sentences" do + it "returns sentences that match the QUESTION_PATTERN" do + expect(subject.question_sentences).to eq(stubbed_sentences) + end + + context "when a sentence does not contain a question" do + let(:stubbed_sentences) { ["This is not a question.", "What do you mean?"] } + + it "filters out non-question sentences" do + expect(subject.question_sentences).to eq(["What do you mean?"]) + end + end + end + + describe "#question_count" do + it "returns the total count of questions, including multi-question sentences" do + allow(subject).to receive(:sentences).and_return([ + "Who are you?", + "What do you want, and why?" + ]) + expect(subject.question_count).to eq(3) + end + + context "when no questions are present" do + let(:stubbed_sentences) { ["This is a statement.", "Another statement."] } + + it "returns zero" do + expect(subject.question_count).to eq(0) + end + end + end +end diff --git a/services/QuillLMS/engines/evidence/spec/services/evidence/gen_ai/evaluation/direction_count/checker_spec.rb b/services/QuillLMS/engines/evidence/spec/services/evidence/gen_ai/evaluation/direction_count/checker_spec.rb new file mode 100644 index 00000000000..3a1a1d962b7 --- /dev/null +++ b/services/QuillLMS/engines/evidence/spec/services/evidence/gen_ai/evaluation/direction_count/checker_spec.rb @@ -0,0 +1,37 @@ +require 'rails_helper' + +describe Evidence::GenAI::Evaluation::DirectionCount::Checker do + let(:entry) { "" } + let(:directions_error) { 'two_directions' } + + subject { described_class.run(feedback:, entry:) } + + describe "#run" do + context "when feedback has one or fewer questions" do + let(:feedback) { "What is your name?" } + + it "returns an optimal result" do + expect(subject.optimal).to be true + expect(subject.errors).to be_empty + end + end + + context "when feedback has more than one question" do + let(:feedback) { "What is your name? How are you?" } + + it "returns a suboptimal result with the 'two_directions' error" do + expect(subject.optimal).to be false + expect(subject.errors).to eq([directions_error]) + end + end + + context "combo questions" do + let(:feedback) { "Hello! What is your name, and how are you?" } + + it "returns a suboptimal result with the 'two_directions' error" do + expect(subject.optimal).to be false + expect(subject.errors).to eq([directions_error]) + end + end + end +end diff --git a/services/QuillLMS/engines/evidence/spec/services/evidence/gen_ai/evaluation/feedback_evaluator_spec.rb b/services/QuillLMS/engines/evidence/spec/services/evidence/gen_ai/evaluation/feedback_evaluator_spec.rb new file mode 100644 index 00000000000..feb016d572f --- /dev/null +++ b/services/QuillLMS/engines/evidence/spec/services/evidence/gen_ai/evaluation/feedback_evaluator_spec.rb @@ -0,0 +1,60 @@ +require 'rails_helper' + +describe Evidence::GenAI::Evaluation::FeedbackEvaluator do + let(:feedback) { "This is some feedback text." } + let(:entry) { "This is the original entry." } + let(:tone_checker) { class_double("Checker", run: tone_result) } + let(:grammar_checker) { class_double("Checker", run: grammar_result) } + let(:verbose_checker) { class_double("Checker", run: verbose_result) } + let(:direction_count_checker) { class_double("Checker", run: direction_count_result) } + + let(:tone_result) { double(optimal: true, errors: []) } + let(:grammar_result) { double(optimal: true, errors: []) } + let(:verbose_result) { double(optimal: false, errors: [verbose_error]) } + let(:direction_count_result) { double(optimal: true, errors: []) } + let(:verbose_error) { 'verbose_error' } + let(:tone_error) { 'tone_error' } + + + describe "#run" do + before do + stub_const("#{described_class}::CHECKS", [tone_checker, grammar_checker, verbose_checker, direction_count_checker]) + end + + subject { described_class.run(feedback:, entry:) } + + context "when all checks are optimal" do + let(:verbose_result) { double(optimal: true, errors: []) } + + it "returns an optimal result with no errors" do + expect(subject.optimal).to be true + expect(subject.errors).to be_empty + end + end + + context "when a checks is not optimal" do + it "returns a result with errors from failing checks" do + expect(subject.optimal).to be false + expect(subject.errors).to eq([verbose_error]) + end + end + + context "when multiple checks are not optimal" do + let(:tone_result) { double(optimal: false, errors: [tone_error]) } + + it "returns a result with errors from failing checks" do + expect(subject.optimal).to be false + expect(subject.errors.sort).to eq([tone_error, verbose_error]) + end + end + + context "one check with multiple errors" do + let(:verbose_result) { double(optimal: false, errors: [tone_error, verbose_error]) } + + it "returns a result with errors from failing checks" do + expect(subject.optimal).to be false + expect(subject.errors.sort).to eq([tone_error, verbose_error]) + end + end + end +end diff --git a/services/QuillLMS/engines/evidence/spec/services/evidence/gen_ai/evaluation/grammar/checker_spec.rb b/services/QuillLMS/engines/evidence/spec/services/evidence/gen_ai/evaluation/grammar/checker_spec.rb new file mode 100644 index 00000000000..e810618e9a4 --- /dev/null +++ b/services/QuillLMS/engines/evidence/spec/services/evidence/gen_ai/evaluation/grammar/checker_spec.rb @@ -0,0 +1,46 @@ +require 'rails_helper' + +describe Evidence::GenAI::Evaluation::Grammar::Checker do + let(:entry) { "" } + let(:grammar_error) { 'grammar' } + + subject { described_class.run(feedback:, entry:) } + + describe "#run" do + context "when feedback does not contain grammar issues" do + let(:feedback) { "This is a well-written sentence." } + + it "returns an optimal result" do + expect(subject.optimal).to be true + expect(subject.errors).to be_empty + end + end + + context "when feedback contains a grammar issue" do + let(:feedback) { "There is a grammar issue here." } + + it "returns a suboptimal result with the 'grammar' error" do + expect(subject.optimal).to be false + expect(subject.errors).to eq([grammar_error]) + end + end + + context "when feedback contains spelling or punctuation issues" do + let(:feedback) { "This sentence has a punctuation problem, like, too many commas." } + + it "returns a suboptimal result with the 'grammar' error" do + expect(subject.optimal).to be false + expect(subject.errors).to eq([grammar_error]) + end + end + + context "when feedback contains multiple grammar-related issues" do + let(:feedback) { "Grammar and punctuation are both problematic." } + + it "returns a suboptimal result with the 'grammar' error" do + expect(subject.optimal).to be false + expect(subject.errors).to eq([grammar_error]) + end + end + end +end diff --git a/services/QuillLMS/engines/evidence/spec/services/evidence/gen_ai/evaluation/tone/checker_spec.rb b/services/QuillLMS/engines/evidence/spec/services/evidence/gen_ai/evaluation/tone/checker_spec.rb new file mode 100644 index 00000000000..3737dc67a3b --- /dev/null +++ b/services/QuillLMS/engines/evidence/spec/services/evidence/gen_ai/evaluation/tone/checker_spec.rb @@ -0,0 +1,64 @@ +require 'rails_helper' + +describe Evidence::GenAI::Evaluation::Tone::Checker do + let(:entry) { "" } + let(:tone_error) { 'tone' } + + subject { described_class.run(feedback:, entry:) } + + describe "#run" do + context "when feedback does not contain tone issues" do + let(:feedback) { "This is a neutral and well-phrased response." } + + it "returns an optimal result" do + expect(subject.optimal).to be true + expect(subject.errors).to be_empty + end + end + + context "when feedback contains tone issues" do + let(:feedback) { "That's incorrect." } + + it "returns a suboptimal result with the 'tone' error" do + expect(subject.optimal).to be false + expect(subject.errors).to eq([tone_error]) + end + end + + context "when feedback suggests improvement" do + let(:feedback) { "To make your response better, consider..." } + + it "returns a suboptimal result with the 'tone' error" do + expect(subject.optimal).to be false + expect(subject.errors).to eq([tone_error]) + end + end + + context "when feedback mentions off-topic or unrelated content" do + let(:feedback) { "This response is unrelated to the question." } + + it "returns a suboptimal result with the 'tone' error" do + expect(subject.optimal).to be false + expect(subject.errors).to eq([tone_error]) + end + end + + context "when feedback contains polite suggestions" do + let(:feedback) { "Please revise your sentence." } + + it "returns a suboptimal result with the 'tone' error" do + expect(subject.optimal).to be false + expect(subject.errors).to eq([tone_error]) + end + end + + context "when feedback contains combinations of flagged phrases" do + let(:feedback) { "That's incorrect. To make your response better..." } + + it "returns a suboptimal result with the 'tone' error" do + expect(subject.optimal).to be false + expect(subject.errors).to eq([tone_error]) + end + end + end +end diff --git a/services/QuillLMS/engines/evidence/spec/services/evidence/gen_ai/evaluation/verbose/checker_spec.rb b/services/QuillLMS/engines/evidence/spec/services/evidence/gen_ai/evaluation/verbose/checker_spec.rb new file mode 100644 index 00000000000..48b5a6ca70e --- /dev/null +++ b/services/QuillLMS/engines/evidence/spec/services/evidence/gen_ai/evaluation/verbose/checker_spec.rb @@ -0,0 +1,55 @@ +require 'rails_helper' + +describe Evidence::GenAI::Evaluation::Verbose::Checker do + let(:entry) { "" } + let(:verbose_error) { 'verbose' } + + subject { described_class.run(feedback:, entry:) } + + describe "#run" do + context "when feedback is within length limits" do + let(:feedback) { "This is a short and concise response with three sentences. Here is one. Here is another." } + + it "returns an optimal result" do + expect(subject.optimal).to be true + expect(subject.errors).to be_empty + end + end + + context "when feedback exceeds the maximum sentence count" do + let(:feedback) { "Sentence one. Sentence two. Sentence three. Sentence four. Sentence five." } + + it "returns a suboptimal result with the 'verbose' error" do + expect(subject.optimal).to be false + expect(subject.errors).to eq([verbose_error]) + end + end + + context "when feedback has the maximum sentence count but exceeds the size limit" do + let(:feedback) { "Sentence one is quite long and detailed, adding many characters to the overall size. Sentence two also follows this pattern with a great deal of verbosity. Sentence three is similarly verbose, providing extensive and unnecessary detail. Sentence four finishes with excessive content, exceeding the character limit." } + + it "returns a suboptimal result with the 'verbose' error" do + expect(subject.optimal).to be false + expect(subject.errors).to eq([verbose_error]) + end + end + + context "when feedback has the maximum sentence count and is within the size limit" do + let(:feedback) { "Short one. Short two. Short three. Short four." } + + it "returns an optimal result" do + expect(subject.optimal).to be true + expect(subject.errors).to be_empty + end + end + + context "when feedback has fewer than the maximum sentence count but is verbose" do + let(:feedback) { "This response has fewer than the maximum sentence count, but it is still very verbose due to the amount of content in each sentence, resulting in an overly lengthy response even though there are fewer sentences overall." } + + it "returns an optimal result" do + expect(subject.optimal).to be true + expect(subject.errors).to be_empty + end + end + end +end From 9c6b14614a738db7b83d3dedcd31ed11ebf7ca3b Mon Sep 17 00:00:00 2001 From: Daniel Drabik Date: Thu, 23 Jan 2025 11:03:25 -0500 Subject: [PATCH 21/51] Lint --- .../evaluation/direction_count/checker.rb | 1 - .../evaluation/misacknowledgement/checker.rb | 2 +- .../misacknowledgement/prompt_builder.rb | 11 ++-- .../gen_ai/evaluation/tone/checker.rb | 2 +- .../gen_ai/evaluation/verbose/checker.rb | 2 +- .../evidence/app/services/evidence/scalpel.rb | 60 +++++++++---------- .../evidence/lib/core_ext/extensions.rb | 2 + .../lib/core_ext/string_extensions.rb | 3 +- .../lib/core_ext/string_extensions_spec.rb | 44 +++++++------- .../direction_count/checker_spec.rb | 20 ++++--- .../evaluation/feedback_evaluator_spec.rb | 33 +++++----- .../gen_ai/evaluation/grammar/checker_spec.rb | 24 ++++---- .../gen_ai/evaluation/tone/checker_spec.rb | 28 +++++---- .../gen_ai/evaluation/verbose/checker_spec.rb | 32 +++++----- .../spec/services/evidence/scalpel_spec.rb | 32 +++++----- .../QuillLMS/lib/tasks/gen_ai_evaluation.thor | 2 +- 16 files changed, 154 insertions(+), 144 deletions(-) diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/direction_count/checker.rb b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/direction_count/checker.rb index f924352fd43..5bf984dd8a8 100644 --- a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/direction_count/checker.rb +++ b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/direction_count/checker.rb @@ -5,7 +5,6 @@ module GenAI module Evaluation module DirectionCount class Checker < Evaluation::BaseChecker - def run return suboptimal if feedback.question_count > 1 diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/checker.rb b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/checker.rb index db7f8107c6c..005b82b030d 100644 --- a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/checker.rb +++ b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/checker.rb @@ -21,7 +21,7 @@ def run private def model = CHAT_API::LARGE_MODEL private def acknowledgement - feedback.sentences.find {|s| !s.match?(CLEAR_RESPONSE_REGEX)} + feedback.sentences.find { |s| !s.match?(CLEAR_RESPONSE_REGEX) } end end end diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompt_builder.rb b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompt_builder.rb index f404a948159..a2b1e39ef24 100644 --- a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompt_builder.rb +++ b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompt_builder.rb @@ -11,22 +11,21 @@ class PromptBuilder < ApplicationService TEMPLATE = '2025_01_17_iteration_with_examples.md' DELIMITER = ' :: ' - Example = Data.define(:entry, :feedback, :optimal) EXAMPLES = [ Example.new('the ocean is deep', "It's true that the ocean is deep.", true), - Example.new('the ocean is deep', "Instead of explaining that the ocean is deep, write about the issue with ocean pollution.", true), + Example.new('the ocean is deep', 'Instead of explaining that the ocean is deep, write about the issue with ocean pollution.', true), Example.new('the war was long and hard', "It's true, the war was long.", true), Example.new('they went to the ballgame', "It's true that Tom and his friends went to the ballgame.", true), Example.new('there were many people involved in the battle', "It's true that the battle involved many people.", true), - Example.new('they wanted a manned mission to space', "Instead of writing about NASA wanted a manned mission to space, focus on the budget issues in the text.", true), + Example.new('they wanted a manned mission to space', 'Instead of writing about NASA wanted a manned mission to space, focus on the budget issues in the text.', true), Example.new('the war was long and hard', "It's true war was long.", true), Example.new('the pasta is made with wheat flour and water', "It's true that pasta has many ingredients, but instead write about the cooking method.", true), - Example.new('the pasta is made with wheat flour and water', "Instead of talking about where pasta was invented, write about the cooking method.", false), + Example.new('the pasta is made with wheat flour and water', 'Instead of talking about where pasta was invented, write about the cooking method.', false), Example.new('the boys were scared.', "It's true that the boys were small and frightened.", false), Example.new('some gibberish.', "It's true pizza is delicious.", false), - Example.new('the pasta is made with wheat flour and water', "Instead of talking about where pasta was invented, write about the cooking method.", false) + Example.new('the pasta is made with wheat flour and water', 'Instead of talking about where pasta was invented, write about the cooking method.', false) ] attr_reader :entry @@ -45,7 +44,7 @@ def initialize(entry:) private def delimiter = DELIMITER private def examples_table = markdown_table_rows(examples) - private def examples = EXAMPLES.map{|e| [[e.entry,delimiter,e.feedback].join(''), e.optimal]} + private def examples = EXAMPLES.map { |e| [[e.entry, delimiter, e.feedback].join(''), e.optimal] } def run = template % template_variables diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/tone/checker.rb b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/tone/checker.rb index 854b3f19f25..71626472b8e 100644 --- a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/tone/checker.rb +++ b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/tone/checker.rb @@ -5,7 +5,7 @@ module GenAI module Evaluation module Tone class Checker < Evaluation::BaseChecker - TONE_REGEX = /(That\'s incorrect|To make your response better|unrelated|off-topic|Can you|You\'re right|You didn\'t|Please|sentence starts with)/ + TONE_REGEX = /(That's incorrect|To make your response better|unrelated|off-topic|Can you|You're right|You didn't|Please|sentence starts with)/ def run return suboptimal if feedback.match?(TONE_REGEX) diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/verbose/checker.rb b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/verbose/checker.rb index a9f9d4bf823..26cd7920ffd 100644 --- a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/verbose/checker.rb +++ b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/verbose/checker.rb @@ -18,7 +18,7 @@ def run return true if feedback.sentence_count > MAX_SENTENCES feedback.sentence_count == MAX_SENTENCES && - feedback.size > MAX_SIZE + feedback.size > MAX_SIZE end private def suboptimal = Result.new(optimal: false, errors: ['verbose']) diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/scalpel.rb b/services/QuillLMS/engines/evidence/app/services/evidence/scalpel.rb index 727b733592d..0b768bd2f9d 100644 --- a/services/QuillLMS/engines/evidence/app/services/evidence/scalpel.rb +++ b/services/QuillLMS/engines/evidence/app/services/evidence/scalpel.rb @@ -1,4 +1,5 @@ # frozen_string_literal: true + # # File copied (and edited) from a 1 file gem at https://github.com/louismullie/scalpel # since RubyGems doesn't have the last version and there were some bugs / needed better test coverage @@ -9,45 +10,43 @@ # .!? being used for other purposes than marking a # full stop before naively segmenting the text. class Evidence::Scalpel - # Current version. VERSION = '0.2.3' # Segment a text using the Scalpel algorithm. # This will eventually be ported to a gem. def self.cut(text) - # Get a copy of the string. text = text.dup.to_s # Remove composite abbreviations. text.gsub!('et al.', 'etal.') # Remove list enders before a comma. - text.gsub!(/(etc|etal)(\.)(\,)/) { $1 + '&|&' + $3} + text.gsub!(/(etc|etal)(\.)(,)/) { ::Regexp.last_match(1) + '&|&' + ::Regexp.last_match(3) } # Remove suspension points. text.gsub!('...', '&;&.') # Remove floating point numbers. - text.gsub!(/([0-9]+)\.([0-9]+)/) { $1 + '&@&' + $2 } + text.gsub!(/([0-9]+)\.([0-9]+)/) { ::Regexp.last_match(1) + '&@&' + ::Regexp.last_match(2) } # Handle floats without leading zero. - text.gsub!(/\s\.([0-9]+)/) { ' &#&' + $1 } + text.gsub!(/\s\.([0-9]+)/) { ' &#&' + ::Regexp.last_match(1) } # Remove abbreviations. text.gsub!(/(?:[A-Za-z]\.){2,}/) { |abbr| abbr.gsub('.', '&-&') } # Remove initials. - text.gsub!(/(?:[A-Z]\.)/) {|abbr| abbr.gsub('.', '&£&') } + text.gsub!(/(?:[A-Z]\.)/) { |abbr| abbr.gsub('.', '&£&') } # Remove titles. text.gsub!(/[A-Z][a-z]{1,2}\./) { |title| title.gsub('.', '&*&') } # Unstick sentences from each other. - text.gsub!(/([^.?!]\.|\!|\?)([^\s"'])/) { $1 + ' ' + $2 } + text.gsub!(/([^.?!]\.|!|\?)([^\s"'])/) { ::Regexp.last_match(1) + ' ' + ::Regexp.last_match(2) } # Remove sentence enders next to quotes. - text.gsub!(/'([.?!])\s?"/) { '&^&' + $1 } - text.gsub!(/'([.?!])\s?”/) { '&*&' + $1 } - text.gsub!(/([.?!])\s?”/) { '&=&' + $1 } - text.gsub!(/([.?!])\s?'"/) { '&,&' + $1 } - text.gsub!(/([.?!])\s?'/) { '&%&' + $1 } - text.gsub!(/([.?!])\s?"/) { '&$&' + $1 } + text.gsub!(/'([.?!])\s?"/) { '&^&' + ::Regexp.last_match(1) } + text.gsub!(/'([.?!])\s?”/) { '&*&' + ::Regexp.last_match(1) } + text.gsub!(/([.?!])\s?”/) { '&=&' + ::Regexp.last_match(1) } + text.gsub!(/([.?!])\s?'"/) { '&,&' + ::Regexp.last_match(1) } + text.gsub!(/([.?!])\s?'/) { '&%&' + ::Regexp.last_match(1) } + text.gsub!(/([.?!])\s?"/) { '&$&' + ::Regexp.last_match(1) } # Remove sentence enders before parens. - text.gsub!(/([.])\s?(\))/) { '&€1&' + $2 } - text.gsub!(/([?])\s?(\))/) { '&€2&' + $2 } - text.gsub!(/([!])\s?(\))/) { '&€3&' + $2 } + text.gsub!(/([.])\s?(\))/) { '&€1&' + ::Regexp.last_match(2) } + text.gsub!(/([?])\s?(\))/) { '&€2&' + ::Regexp.last_match(2) } + text.gsub!(/([!])\s?(\))/) { '&€3&' + ::Regexp.last_match(2) } # Split on any sentence ender. sentences = text.split(/([.!?])/) @@ -61,12 +60,13 @@ def self.cut(text) new_sents.each do |sentence| # Skip whitespace zones. next if sentence.strip == '' + # Repair composite abbreviations. sentence.gsub!('etal', 'et al') # Repair list enders before a comma. - sentence.gsub!("&\|&", ".") + sentence.gsub!("&\|&", '.') # Repair initials. - sentence.gsub!("&£&", ".") + sentence.gsub!('&£&', '.') # Repair abbreviations. sentence.gsub!('&-&', '.') # Repair titles. @@ -74,26 +74,24 @@ def self.cut(text) # Repair suspension points. sentence.gsub!('&;&.', '...') # Repair floats. - sentence.gsub!(/([0-9]+)&@&([0-9]+)/) { $1 + '.' + $2 } + sentence.gsub!(/([0-9]+)&@&([0-9]+)/) { ::Regexp.last_match(1) + '.' + ::Regexp.last_match(2) } # Repair quotes with sentence enders - sentence.gsub!(/&=&([.!?])/) { $1 + '”' } - sentence.gsub!(/&,&([.!?])/) { $1 + "'\"" } - sentence.gsub!(/&%&([.!?])/) { $1 + "'" } - sentence.gsub!(/&\^&([.?!])/) { "'" + $1 + '"' } - sentence.gsub!(/&\*&([.?!])/) { "'" + $1 + '”' } - sentence.gsub!(/&\$&([.!?])/) { $1 + '"' } + sentence.gsub!(/&=&([.!?])/) { ::Regexp.last_match(1) + '”' } + sentence.gsub!(/&,&([.!?])/) { ::Regexp.last_match(1) + "'\"" } + sentence.gsub!(/&%&([.!?])/) { ::Regexp.last_match(1) + "'" } + sentence.gsub!(/&\^&([.?!])/) { "'" + ::Regexp.last_match(1) + '"' } + sentence.gsub!(/&\*&([.?!])/) { "'" + ::Regexp.last_match(1) + '”' } + sentence.gsub!(/&\$&([.!?])/) { ::Regexp.last_match(1) + '"' } # Repair parens with sentence enders - sentence.gsub!("&€1&", ".") - sentence.gsub!("&€2&", "?") - sentence.gsub!("&€3&", "!") + sentence.gsub!('&€1&', '.') + sentence.gsub!('&€2&', '?') + sentence.gsub!('&€3&', '!') # Repair floats without leading zeros. - sentence.gsub!(/&#&([0-9]+)/) { '.' + $1 } + sentence.gsub!(/&#&([0-9]+)/) { '.' + ::Regexp.last_match(1) } results << sentence.strip end results - end - end diff --git a/services/QuillLMS/engines/evidence/lib/core_ext/extensions.rb b/services/QuillLMS/engines/evidence/lib/core_ext/extensions.rb index e619160ec9d..fd7d225715c 100644 --- a/services/QuillLMS/engines/evidence/lib/core_ext/extensions.rb +++ b/services/QuillLMS/engines/evidence/lib/core_ext/extensions.rb @@ -1 +1,3 @@ +# frozen_string_literal: true + require_relative 'string_extensions' diff --git a/services/QuillLMS/engines/evidence/lib/core_ext/string_extensions.rb b/services/QuillLMS/engines/evidence/lib/core_ext/string_extensions.rb index 200e870e823..f34f93b6a1c 100644 --- a/services/QuillLMS/engines/evidence/lib/core_ext/string_extensions.rb +++ b/services/QuillLMS/engines/evidence/lib/core_ext/string_extensions.rb @@ -16,7 +16,8 @@ class String def sentences = Evidence::Scalpel.cut(self) def sentence_count = sentences.size - def question_sentences = sentences.select {|s| s.match?(QUESTION_PATTERN)} + def question_sentences = sentences.select { |s| s.match?(QUESTION_PATTERN) } + def question_count question_sentences.reduce(0) do |count, sentence| questions_in_sentence = sentence.match?(MULTI_QUESTION_PATTERN) ? 2 : 1 diff --git a/services/QuillLMS/engines/evidence/spec/lib/core_ext/string_extensions_spec.rb b/services/QuillLMS/engines/evidence/spec/lib/core_ext/string_extensions_spec.rb index 2048b8435a3..d89884db3c0 100644 --- a/services/QuillLMS/engines/evidence/spec/lib/core_ext/string_extensions_spec.rb +++ b/services/QuillLMS/engines/evidence/spec/lib/core_ext/string_extensions_spec.rb @@ -1,8 +1,10 @@ +# frozen_string_literal: true + require 'rails_helper' -describe "String#sentences and related methods" do - let(:input_string) { "Who are you? What do you want?" } - let(:stubbed_sentences) { ["Who are you?", "What do you want?"] } +describe 'String#sentences and related methods' do + let(:input_string) { 'Who are you? What do you want?' } + let(:stubbed_sentences) { ['Who are you?', 'What do you want?'] } before do allow(Evidence::Scalpel).to receive(:cut).and_return(stubbed_sentences) @@ -10,50 +12,50 @@ subject { input_string } - describe "#sentences" do - it "returns an array of sentences using the Scalpel.cut method" do + describe '#sentences' do + it 'returns an array of sentences using the Scalpel.cut method' do expect(subject.sentences).to eq(stubbed_sentences) end - it "calls Scalpel.cut with the input string" do + it 'calls Scalpel.cut with the input string' do subject.sentences expect(Evidence::Scalpel).to have_received(:cut).with(input_string) end end - describe "#sentence_count" do - it "returns the number of sentences" do + describe '#sentence_count' do + it 'returns the number of sentences' do expect(subject.sentence_count).to eq(stubbed_sentences.size) end end - describe "#question_sentences" do - it "returns sentences that match the QUESTION_PATTERN" do + describe '#question_sentences' do + it 'returns sentences that match the QUESTION_PATTERN' do expect(subject.question_sentences).to eq(stubbed_sentences) end - context "when a sentence does not contain a question" do - let(:stubbed_sentences) { ["This is not a question.", "What do you mean?"] } + context 'when a sentence does not contain a question' do + let(:stubbed_sentences) { ['This is not a question.', 'What do you mean?'] } - it "filters out non-question sentences" do - expect(subject.question_sentences).to eq(["What do you mean?"]) + it 'filters out non-question sentences' do + expect(subject.question_sentences).to eq(['What do you mean?']) end end end - describe "#question_count" do - it "returns the total count of questions, including multi-question sentences" do + describe '#question_count' do + it 'returns the total count of questions, including multi-question sentences' do allow(subject).to receive(:sentences).and_return([ - "Who are you?", - "What do you want, and why?" + 'Who are you?', + 'What do you want, and why?' ]) expect(subject.question_count).to eq(3) end - context "when no questions are present" do - let(:stubbed_sentences) { ["This is a statement.", "Another statement."] } + context 'when no questions are present' do + let(:stubbed_sentences) { ['This is a statement.', 'Another statement.'] } - it "returns zero" do + it 'returns zero' do expect(subject.question_count).to eq(0) end end diff --git a/services/QuillLMS/engines/evidence/spec/services/evidence/gen_ai/evaluation/direction_count/checker_spec.rb b/services/QuillLMS/engines/evidence/spec/services/evidence/gen_ai/evaluation/direction_count/checker_spec.rb index 3a1a1d962b7..266fadff3af 100644 --- a/services/QuillLMS/engines/evidence/spec/services/evidence/gen_ai/evaluation/direction_count/checker_spec.rb +++ b/services/QuillLMS/engines/evidence/spec/services/evidence/gen_ai/evaluation/direction_count/checker_spec.rb @@ -1,23 +1,25 @@ +# frozen_string_literal: true + require 'rails_helper' describe Evidence::GenAI::Evaluation::DirectionCount::Checker do - let(:entry) { "" } + let(:entry) { '' } let(:directions_error) { 'two_directions' } subject { described_class.run(feedback:, entry:) } - describe "#run" do - context "when feedback has one or fewer questions" do - let(:feedback) { "What is your name?" } + describe '#run' do + context 'when feedback has one or fewer questions' do + let(:feedback) { 'What is your name?' } - it "returns an optimal result" do + it 'returns an optimal result' do expect(subject.optimal).to be true expect(subject.errors).to be_empty end end - context "when feedback has more than one question" do - let(:feedback) { "What is your name? How are you?" } + context 'when feedback has more than one question' do + let(:feedback) { 'What is your name? How are you?' } it "returns a suboptimal result with the 'two_directions' error" do expect(subject.optimal).to be false @@ -25,8 +27,8 @@ end end - context "combo questions" do - let(:feedback) { "Hello! What is your name, and how are you?" } + context 'combo questions' do + let(:feedback) { 'Hello! What is your name, and how are you?' } it "returns a suboptimal result with the 'two_directions' error" do expect(subject.optimal).to be false diff --git a/services/QuillLMS/engines/evidence/spec/services/evidence/gen_ai/evaluation/feedback_evaluator_spec.rb b/services/QuillLMS/engines/evidence/spec/services/evidence/gen_ai/evaluation/feedback_evaluator_spec.rb index feb016d572f..6de1b2fcd51 100644 --- a/services/QuillLMS/engines/evidence/spec/services/evidence/gen_ai/evaluation/feedback_evaluator_spec.rb +++ b/services/QuillLMS/engines/evidence/spec/services/evidence/gen_ai/evaluation/feedback_evaluator_spec.rb @@ -1,12 +1,14 @@ +# frozen_string_literal: true + require 'rails_helper' describe Evidence::GenAI::Evaluation::FeedbackEvaluator do - let(:feedback) { "This is some feedback text." } - let(:entry) { "This is the original entry." } - let(:tone_checker) { class_double("Checker", run: tone_result) } - let(:grammar_checker) { class_double("Checker", run: grammar_result) } - let(:verbose_checker) { class_double("Checker", run: verbose_result) } - let(:direction_count_checker) { class_double("Checker", run: direction_count_result) } + let(:feedback) { 'This is some feedback text.' } + let(:entry) { 'This is the original entry.' } + let(:tone_checker) { class_double(run: tone_result) } + let(:grammar_checker) { class_double(run: grammar_result) } + let(:verbose_checker) { class_double(run: verbose_result) } + let(:direction_count_checker) { class_double(run: direction_count_result) } let(:tone_result) { double(optimal: true, errors: []) } let(:grammar_result) { double(optimal: true, errors: []) } @@ -15,43 +17,42 @@ let(:verbose_error) { 'verbose_error' } let(:tone_error) { 'tone_error' } - - describe "#run" do + describe '#run' do before do stub_const("#{described_class}::CHECKS", [tone_checker, grammar_checker, verbose_checker, direction_count_checker]) end subject { described_class.run(feedback:, entry:) } - context "when all checks are optimal" do + context 'when all checks are optimal' do let(:verbose_result) { double(optimal: true, errors: []) } - it "returns an optimal result with no errors" do + it 'returns an optimal result with no errors' do expect(subject.optimal).to be true expect(subject.errors).to be_empty end end - context "when a checks is not optimal" do - it "returns a result with errors from failing checks" do + context 'when a checks is not optimal' do + it 'returns a result with errors from failing checks' do expect(subject.optimal).to be false expect(subject.errors).to eq([verbose_error]) end end - context "when multiple checks are not optimal" do + context 'when multiple checks are not optimal' do let(:tone_result) { double(optimal: false, errors: [tone_error]) } - it "returns a result with errors from failing checks" do + it 'returns a result with errors from failing checks' do expect(subject.optimal).to be false expect(subject.errors.sort).to eq([tone_error, verbose_error]) end end - context "one check with multiple errors" do + context 'one check with multiple errors' do let(:verbose_result) { double(optimal: false, errors: [tone_error, verbose_error]) } - it "returns a result with errors from failing checks" do + it 'returns a result with errors from failing checks' do expect(subject.optimal).to be false expect(subject.errors.sort).to eq([tone_error, verbose_error]) end diff --git a/services/QuillLMS/engines/evidence/spec/services/evidence/gen_ai/evaluation/grammar/checker_spec.rb b/services/QuillLMS/engines/evidence/spec/services/evidence/gen_ai/evaluation/grammar/checker_spec.rb index e810618e9a4..acb890771ef 100644 --- a/services/QuillLMS/engines/evidence/spec/services/evidence/gen_ai/evaluation/grammar/checker_spec.rb +++ b/services/QuillLMS/engines/evidence/spec/services/evidence/gen_ai/evaluation/grammar/checker_spec.rb @@ -1,23 +1,25 @@ +# frozen_string_literal: true + require 'rails_helper' describe Evidence::GenAI::Evaluation::Grammar::Checker do - let(:entry) { "" } + let(:entry) { '' } let(:grammar_error) { 'grammar' } subject { described_class.run(feedback:, entry:) } - describe "#run" do - context "when feedback does not contain grammar issues" do - let(:feedback) { "This is a well-written sentence." } + describe '#run' do + context 'when feedback does not contain grammar issues' do + let(:feedback) { 'This is a well-written sentence.' } - it "returns an optimal result" do + it 'returns an optimal result' do expect(subject.optimal).to be true expect(subject.errors).to be_empty end end - context "when feedback contains a grammar issue" do - let(:feedback) { "There is a grammar issue here." } + context 'when feedback contains a grammar issue' do + let(:feedback) { 'There is a grammar issue here.' } it "returns a suboptimal result with the 'grammar' error" do expect(subject.optimal).to be false @@ -25,8 +27,8 @@ end end - context "when feedback contains spelling or punctuation issues" do - let(:feedback) { "This sentence has a punctuation problem, like, too many commas." } + context 'when feedback contains spelling or punctuation issues' do + let(:feedback) { 'This sentence has a punctuation problem, like, too many commas.' } it "returns a suboptimal result with the 'grammar' error" do expect(subject.optimal).to be false @@ -34,8 +36,8 @@ end end - context "when feedback contains multiple grammar-related issues" do - let(:feedback) { "Grammar and punctuation are both problematic." } + context 'when feedback contains multiple grammar-related issues' do + let(:feedback) { 'Grammar and punctuation are both problematic.' } it "returns a suboptimal result with the 'grammar' error" do expect(subject.optimal).to be false diff --git a/services/QuillLMS/engines/evidence/spec/services/evidence/gen_ai/evaluation/tone/checker_spec.rb b/services/QuillLMS/engines/evidence/spec/services/evidence/gen_ai/evaluation/tone/checker_spec.rb index 3737dc67a3b..d5f547bd52a 100644 --- a/services/QuillLMS/engines/evidence/spec/services/evidence/gen_ai/evaluation/tone/checker_spec.rb +++ b/services/QuillLMS/engines/evidence/spec/services/evidence/gen_ai/evaluation/tone/checker_spec.rb @@ -1,22 +1,24 @@ +# frozen_string_literal: true + require 'rails_helper' describe Evidence::GenAI::Evaluation::Tone::Checker do - let(:entry) { "" } + let(:entry) { '' } let(:tone_error) { 'tone' } subject { described_class.run(feedback:, entry:) } - describe "#run" do - context "when feedback does not contain tone issues" do - let(:feedback) { "This is a neutral and well-phrased response." } + describe '#run' do + context 'when feedback does not contain tone issues' do + let(:feedback) { 'This is a neutral and well-phrased response.' } - it "returns an optimal result" do + it 'returns an optimal result' do expect(subject.optimal).to be true expect(subject.errors).to be_empty end end - context "when feedback contains tone issues" do + context 'when feedback contains tone issues' do let(:feedback) { "That's incorrect." } it "returns a suboptimal result with the 'tone' error" do @@ -25,8 +27,8 @@ end end - context "when feedback suggests improvement" do - let(:feedback) { "To make your response better, consider..." } + context 'when feedback suggests improvement' do + let(:feedback) { 'To make your response better, consider...' } it "returns a suboptimal result with the 'tone' error" do expect(subject.optimal).to be false @@ -34,8 +36,8 @@ end end - context "when feedback mentions off-topic or unrelated content" do - let(:feedback) { "This response is unrelated to the question." } + context 'when feedback mentions off-topic or unrelated content' do + let(:feedback) { 'This response is unrelated to the question.' } it "returns a suboptimal result with the 'tone' error" do expect(subject.optimal).to be false @@ -43,8 +45,8 @@ end end - context "when feedback contains polite suggestions" do - let(:feedback) { "Please revise your sentence." } + context 'when feedback contains polite suggestions' do + let(:feedback) { 'Please revise your sentence.' } it "returns a suboptimal result with the 'tone' error" do expect(subject.optimal).to be false @@ -52,7 +54,7 @@ end end - context "when feedback contains combinations of flagged phrases" do + context 'when feedback contains combinations of flagged phrases' do let(:feedback) { "That's incorrect. To make your response better..." } it "returns a suboptimal result with the 'tone' error" do diff --git a/services/QuillLMS/engines/evidence/spec/services/evidence/gen_ai/evaluation/verbose/checker_spec.rb b/services/QuillLMS/engines/evidence/spec/services/evidence/gen_ai/evaluation/verbose/checker_spec.rb index 48b5a6ca70e..6b9a9ada11c 100644 --- a/services/QuillLMS/engines/evidence/spec/services/evidence/gen_ai/evaluation/verbose/checker_spec.rb +++ b/services/QuillLMS/engines/evidence/spec/services/evidence/gen_ai/evaluation/verbose/checker_spec.rb @@ -1,23 +1,25 @@ +# frozen_string_literal: true + require 'rails_helper' describe Evidence::GenAI::Evaluation::Verbose::Checker do - let(:entry) { "" } + let(:entry) { '' } let(:verbose_error) { 'verbose' } subject { described_class.run(feedback:, entry:) } - describe "#run" do - context "when feedback is within length limits" do - let(:feedback) { "This is a short and concise response with three sentences. Here is one. Here is another." } + describe '#run' do + context 'when feedback is within length limits' do + let(:feedback) { 'This is a short and concise response with three sentences. Here is one. Here is another.' } - it "returns an optimal result" do + it 'returns an optimal result' do expect(subject.optimal).to be true expect(subject.errors).to be_empty end end - context "when feedback exceeds the maximum sentence count" do - let(:feedback) { "Sentence one. Sentence two. Sentence three. Sentence four. Sentence five." } + context 'when feedback exceeds the maximum sentence count' do + let(:feedback) { 'Sentence one. Sentence two. Sentence three. Sentence four. Sentence five.' } it "returns a suboptimal result with the 'verbose' error" do expect(subject.optimal).to be false @@ -25,8 +27,8 @@ end end - context "when feedback has the maximum sentence count but exceeds the size limit" do - let(:feedback) { "Sentence one is quite long and detailed, adding many characters to the overall size. Sentence two also follows this pattern with a great deal of verbosity. Sentence three is similarly verbose, providing extensive and unnecessary detail. Sentence four finishes with excessive content, exceeding the character limit." } + context 'when feedback has the maximum sentence count but exceeds the size limit' do + let(:feedback) { 'Sentence one is quite long and detailed, adding many characters to the overall size. Sentence two also follows this pattern with a great deal of verbosity. Sentence three is similarly verbose, providing extensive and unnecessary detail. Sentence four finishes with excessive content, exceeding the character limit.' } it "returns a suboptimal result with the 'verbose' error" do expect(subject.optimal).to be false @@ -34,19 +36,19 @@ end end - context "when feedback has the maximum sentence count and is within the size limit" do - let(:feedback) { "Short one. Short two. Short three. Short four." } + context 'when feedback has the maximum sentence count and is within the size limit' do + let(:feedback) { 'Short one. Short two. Short three. Short four.' } - it "returns an optimal result" do + it 'returns an optimal result' do expect(subject.optimal).to be true expect(subject.errors).to be_empty end end - context "when feedback has fewer than the maximum sentence count but is verbose" do - let(:feedback) { "This response has fewer than the maximum sentence count, but it is still very verbose due to the amount of content in each sentence, resulting in an overly lengthy response even though there are fewer sentences overall." } + context 'when feedback has fewer than the maximum sentence count but is verbose' do + let(:feedback) { 'This response has fewer than the maximum sentence count, but it is still very verbose due to the amount of content in each sentence, resulting in an overly lengthy response even though there are fewer sentences overall.' } - it "returns an optimal result" do + it 'returns an optimal result' do expect(subject.optimal).to be true expect(subject.errors).to be_empty end diff --git a/services/QuillLMS/engines/evidence/spec/services/evidence/scalpel_spec.rb b/services/QuillLMS/engines/evidence/spec/services/evidence/scalpel_spec.rb index e4bb8f94ae8..874dbc4e5a8 100644 --- a/services/QuillLMS/engines/evidence/spec/services/evidence/scalpel_spec.rb +++ b/services/QuillLMS/engines/evidence/spec/services/evidence/scalpel_spec.rb @@ -6,31 +6,31 @@ describe '.cut' do it 'handles simple sentences correctly' do text = 'This is a sentence. This is another one!' - result = ["This is a sentence.", "This is another one!"] + result = ['This is a sentence.', 'This is another one!'] expect(described_class.cut(text)).to eq(result) end it 'handles sentence ending abbreviation' do text = 'Dr. Smith went to Washington, etc. It was sunny.' - result = ["Dr. Smith went to Washington, etc.", "It was sunny."] + result = ['Dr. Smith went to Washington, etc.', 'It was sunny.'] expect(described_class.cut(text)).to eq(result) end it 'handles sentence ending abbreviation with et al.' do text = 'Dr. Smith went to Washington, et al. It was sunny.' - result = ["Dr. Smith went to Washington, et al.", "It was sunny."] + result = ['Dr. Smith went to Washington, et al.', 'It was sunny.'] expect(described_class.cut(text)).to eq(result) end it 'handles abbreviations correctly' do text = 'Dr. Smith went to Washington. It was sunny.' - result = ["Dr. Smith went to Washington.", "It was sunny."] + result = ['Dr. Smith went to Washington.', 'It was sunny.'] expect(described_class.cut(text)).to eq(result) end it 'handles e.g. correctly' do - text = "Please bring the necessary supplies, e.g. pens, paper, and books." - result = ["Please bring the necessary supplies, e.g. pens, paper, and books."] + text = 'Please bring the necessary supplies, e.g. pens, paper, and books.' + result = ['Please bring the necessary supplies, e.g. pens, paper, and books.'] expect(described_class.cut(text)).to eq(result) end @@ -49,55 +49,55 @@ it 'handles etc. as a sentence ender' do text = "Please bring the necessary supplies like pens, paper, etc. Also, don't forget your notebook." - result = ["Please bring the necessary supplies like pens, paper, etc.", "Also, don't forget your notebook."] + result = ['Please bring the necessary supplies like pens, paper, etc.', "Also, don't forget your notebook."] expect(described_class.cut(text)).to eq(result) end it 'handles numbers with periods' do text = 'The value is 3.14. That is pi.' - result = ["The value is 3.14.", "That is pi."] + result = ['The value is 3.14.', 'That is pi.'] expect(described_class.cut(text)).to eq(result) end it 'handles ellipses correctly' do text = 'I was thinking... maybe we should go. What do you think?' - result = ["I was thinking...", "maybe we should go.", "What do you think?"] + result = ['I was thinking...', 'maybe we should go.', 'What do you think?'] expect(described_class.cut(text)).to eq(result) end it 'handles titles and initials correctly' do text = 'Mr. J.D. Rockefeller was a famous figure. He lived in the U.S.' - result = ["Mr. J.D. Rockefeller was a famous figure.", "He lived in the U.S."] + result = ['Mr. J.D. Rockefeller was a famous figure.', 'He lived in the U.S.'] expect(described_class.cut(text)).to eq(result) end it 'handles nested quotes correctly' do text = "'Did he say, \"Yes\"?' she asked." - result = ["'Did he say, \"Yes\"?'", "she asked."] + result = ["'Did he say, \"Yes\"?'", 'she asked.'] expect(described_class.cut(text)).to eq(result) end it 'handles punctuation near quotes correctly' do text = "She said, 'Hello.' Then she left." - result = ["She said, 'Hello.'", "Then she left."] + result = ["She said, 'Hello.'", 'Then she left.'] expect(described_class.cut(text)).to eq(result) end it 'handles parentheses with sentence enders correctly' do text = 'He was late (again!). She was upset.' - result = ["He was late (again!).", "She was upset."] + result = ['He was late (again!).', 'She was upset.'] expect(described_class.cut(text)).to eq(result) end it 'handles parentheses within sentence if it has sentence ender' do text = "He was latte (spelling?), but that isn't important. She was upset." - result = ["He was latte (spelling?), but that isn't important.", "She was upset."] + result = ["He was latte (spelling?), but that isn't important.", 'She was upset.'] expect(described_class.cut(text)).to eq(result) end it 'handles extra whitespace correctly' do text = ' Hello world. This is a test. ' - result = ["Hello world.", "This is a test."] + result = ['Hello world.', 'This is a test.'] expect(described_class.cut(text)).to eq(result) end @@ -109,7 +109,7 @@ it 'handles floating point numbers without leading zeros' do text = 'The value was .5. It increased to 1.5.' - result = ["The value was .5.", "It increased to 1.5."] + result = ['The value was .5.', 'It increased to 1.5.'] expect(described_class.cut(text)).to eq(result) end end diff --git a/services/QuillLMS/lib/tasks/gen_ai_evaluation.thor b/services/QuillLMS/lib/tasks/gen_ai_evaluation.thor index 981edefeaf7..8383824d407 100644 --- a/services/QuillLMS/lib/tasks/gen_ai_evaluation.thor +++ b/services/QuillLMS/lib/tasks/gen_ai_evaluation.thor @@ -37,7 +37,7 @@ class GenAIEvaluation < Thor feedback = item.feedback rule = item.rule entry = item.entry - acknowledgement = feedback.sentences.find {|s| !s.match?(CLEAR_RESPONSE_REGEX)} + acknowledgement = feedback.sentences.find { |s| !s.match?(CLEAR_RESPONSE_REGEX) } response = Evidence::GenAI::Evaluation::FeedbackEvaluator.run(feedback:, entry:) optimal_output = response.optimal From cd41170613b004864b95eb419901276580c09425 Mon Sep 17 00:00:00 2001 From: Daniel Drabik Date: Thu, 23 Jan 2025 12:29:16 -0500 Subject: [PATCH 22/51] Lint and small refactors. --- .../evaluation/misacknowledgement/checker.rb | 2 +- .../misacknowledgement/prompt_builder.rb | 2 +- .../evidence/app/services/evidence/scalpel.rb | 46 +++++++++---------- .../lib/core_ext/string_extensions_spec.rb | 22 ++++----- .../evaluation/feedback_evaluator_spec.rb | 8 ++-- 5 files changed, 37 insertions(+), 43 deletions(-) diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/checker.rb b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/checker.rb index 005b82b030d..19a57256b7d 100644 --- a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/checker.rb +++ b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/checker.rb @@ -16,7 +16,7 @@ def run private def suboptimal = Result.new(optimal: false, errors: ['misacknowledgement']) private def delimiter = PromptBuilder::DELIMITER - private def llm_entry = [entry, delimiter, acknowledgement].join('') + private def llm_entry = [entry, delimiter, acknowledgement].join private def system_prompt = Evidence::GenAI::Evaluation::Misacknowledgement::PromptBuilder.run(entry:) private def model = CHAT_API::LARGE_MODEL diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompt_builder.rb b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompt_builder.rb index a2b1e39ef24..9882c9f8f50 100644 --- a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompt_builder.rb +++ b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompt_builder.rb @@ -44,7 +44,7 @@ def initialize(entry:) private def delimiter = DELIMITER private def examples_table = markdown_table_rows(examples) - private def examples = EXAMPLES.map { |e| [[e.entry, delimiter, e.feedback].join(''), e.optimal] } + private def examples = EXAMPLES.map { |e| [[e.entry, delimiter, e.feedback].join, e.optimal] } def run = template % template_variables diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/scalpel.rb b/services/QuillLMS/engines/evidence/app/services/evidence/scalpel.rb index 0b768bd2f9d..1c87e4438f4 100644 --- a/services/QuillLMS/engines/evidence/app/services/evidence/scalpel.rb +++ b/services/QuillLMS/engines/evidence/app/services/evidence/scalpel.rb @@ -11,7 +11,7 @@ # full stop before naively segmenting the text. class Evidence::Scalpel # Current version. - VERSION = '0.2.3' + VERSION = '0.3.0' # Segment a text using the Scalpel algorithm. # This will eventually be ported to a gem. @@ -21,13 +21,13 @@ def self.cut(text) # Remove composite abbreviations. text.gsub!('et al.', 'etal.') # Remove list enders before a comma. - text.gsub!(/(etc|etal)(\.)(,)/) { ::Regexp.last_match(1) + '&|&' + ::Regexp.last_match(3) } + text.gsub!(/(etc|etal)(\.)(,)/) { "#{::Regexp.last_match(1)}&|&#{::Regexp.last_match(3)}" } # Remove suspension points. text.gsub!('...', '&;&.') # Remove floating point numbers. - text.gsub!(/([0-9]+)\.([0-9]+)/) { ::Regexp.last_match(1) + '&@&' + ::Regexp.last_match(2) } + text.gsub!(/([0-9]+)\.([0-9]+)/) { "#{::Regexp.last_match(1)}&@&#{::Regexp.last_match(2)}" } # Handle floats without leading zero. - text.gsub!(/\s\.([0-9]+)/) { ' &#&' + ::Regexp.last_match(1) } + text.gsub!(/\s\.([0-9]+)/) { " &#&#{::Regexp.last_match(1)}" } # Remove abbreviations. text.gsub!(/(?:[A-Za-z]\.){2,}/) { |abbr| abbr.gsub('.', '&-&') } # Remove initials. @@ -35,25 +35,25 @@ def self.cut(text) # Remove titles. text.gsub!(/[A-Z][a-z]{1,2}\./) { |title| title.gsub('.', '&*&') } # Unstick sentences from each other. - text.gsub!(/([^.?!]\.|!|\?)([^\s"'])/) { ::Regexp.last_match(1) + ' ' + ::Regexp.last_match(2) } + text.gsub!(/([^.?!]\.|!|\?)([^\s"'])/) { "#{::Regexp.last_match(1)} #{::Regexp.last_match(2)}" } # Remove sentence enders next to quotes. - text.gsub!(/'([.?!])\s?"/) { '&^&' + ::Regexp.last_match(1) } - text.gsub!(/'([.?!])\s?”/) { '&*&' + ::Regexp.last_match(1) } - text.gsub!(/([.?!])\s?”/) { '&=&' + ::Regexp.last_match(1) } - text.gsub!(/([.?!])\s?'"/) { '&,&' + ::Regexp.last_match(1) } - text.gsub!(/([.?!])\s?'/) { '&%&' + ::Regexp.last_match(1) } - text.gsub!(/([.?!])\s?"/) { '&$&' + ::Regexp.last_match(1) } + text.gsub!(/'([.?!])\s?"/) { "&^&#{::Regexp.last_match(1)}" } + text.gsub!(/'([.?!])\s?”/) { "&*&#{::Regexp.last_match(1)}" } + text.gsub!(/([.?!])\s?”/) { "&=&#{::Regexp.last_match(1)}" } + text.gsub!(/([.?!])\s?'"/) { "&,&#{::Regexp.last_match(1)}" } + text.gsub!(/([.?!])\s?'/) { "&%&#{::Regexp.last_match(1)}" } + text.gsub!(/([.?!])\s?"/) { "&$&#{::Regexp.last_match(1)}" } # Remove sentence enders before parens. - text.gsub!(/([.])\s?(\))/) { '&€1&' + ::Regexp.last_match(2) } - text.gsub!(/([?])\s?(\))/) { '&€2&' + ::Regexp.last_match(2) } - text.gsub!(/([!])\s?(\))/) { '&€3&' + ::Regexp.last_match(2) } + text.gsub!(/([.])\s?(\))/) { "&€1&#{::Regexp.last_match(2)}" } + text.gsub!(/([?])\s?(\))/) { "&€2&#{::Regexp.last_match(2)}" } + text.gsub!(/([!])\s?(\))/) { "&€3&#{::Regexp.last_match(2)}" } # Split on any sentence ender. sentences = text.split(/([.!?])/) new_sents = [] # Join the obtained slices. sentences.each_slice(2) do |slice| - new_sents << slice.join('') + new_sents << slice.join end # Repair the damage we've done. results = [] @@ -74,21 +74,21 @@ def self.cut(text) # Repair suspension points. sentence.gsub!('&;&.', '...') # Repair floats. - sentence.gsub!(/([0-9]+)&@&([0-9]+)/) { ::Regexp.last_match(1) + '.' + ::Regexp.last_match(2) } + sentence.gsub!(/([0-9]+)&@&([0-9]+)/) { "#{::Regexp.last_match(1)}.#{::Regexp.last_match(2)}" } # Repair quotes with sentence enders - sentence.gsub!(/&=&([.!?])/) { ::Regexp.last_match(1) + '”' } - sentence.gsub!(/&,&([.!?])/) { ::Regexp.last_match(1) + "'\"" } - sentence.gsub!(/&%&([.!?])/) { ::Regexp.last_match(1) + "'" } - sentence.gsub!(/&\^&([.?!])/) { "'" + ::Regexp.last_match(1) + '"' } - sentence.gsub!(/&\*&([.?!])/) { "'" + ::Regexp.last_match(1) + '”' } - sentence.gsub!(/&\$&([.!?])/) { ::Regexp.last_match(1) + '"' } + sentence.gsub!(/&=&([.!?])/) { "#{::Regexp.last_match(1)}”" } + sentence.gsub!(/&,&([.!?])/) { "#{::Regexp.last_match(1)}'\"" } + sentence.gsub!(/&%&([.!?])/) { "#{::Regexp.last_match(1)}'" } + sentence.gsub!(/&\^&([.?!])/) { "'#{::Regexp.last_match(1)}\"" } + sentence.gsub!(/&\*&([.?!])/) { "'#{::Regexp.last_match(1)}”" } + sentence.gsub!(/&\$&([.!?])/) { "#{::Regexp.last_match(1)}\"" } # Repair parens with sentence enders sentence.gsub!('&€1&', '.') sentence.gsub!('&€2&', '?') sentence.gsub!('&€3&', '!') # Repair floats without leading zeros. - sentence.gsub!(/&#&([0-9]+)/) { '.' + ::Regexp.last_match(1) } + sentence.gsub!(/&#&([0-9]+)/) { ".#{::Regexp.last_match(1)}" } results << sentence.strip end diff --git a/services/QuillLMS/engines/evidence/spec/lib/core_ext/string_extensions_spec.rb b/services/QuillLMS/engines/evidence/spec/lib/core_ext/string_extensions_spec.rb index d89884db3c0..c278bbd2ac0 100644 --- a/services/QuillLMS/engines/evidence/spec/lib/core_ext/string_extensions_spec.rb +++ b/services/QuillLMS/engines/evidence/spec/lib/core_ext/string_extensions_spec.rb @@ -2,15 +2,10 @@ require 'rails_helper' -describe 'String#sentences and related methods' do - let(:input_string) { 'Who are you? What do you want?' } +describe String do let(:stubbed_sentences) { ['Who are you?', 'What do you want?'] } - before do - allow(Evidence::Scalpel).to receive(:cut).and_return(stubbed_sentences) - end - - subject { input_string } + subject { 'Who are you? What do you want?' } describe '#sentences' do it 'returns an array of sentences using the Scalpel.cut method' do @@ -18,8 +13,9 @@ end it 'calls Scalpel.cut with the input string' do + expect(Evidence::Scalpel).to receive(:cut).with(subject) + subject.sentences - expect(Evidence::Scalpel).to have_received(:cut).with(input_string) end end @@ -35,7 +31,7 @@ end context 'when a sentence does not contain a question' do - let(:stubbed_sentences) { ['This is not a question.', 'What do you mean?'] } + subject { 'This is not a question. What do you mean?' } it 'filters out non-question sentences' do expect(subject.question_sentences).to eq(['What do you mean?']) @@ -44,16 +40,14 @@ end describe '#question_count' do + subject { 'Who are you? What do you want, and why?' } + it 'returns the total count of questions, including multi-question sentences' do - allow(subject).to receive(:sentences).and_return([ - 'Who are you?', - 'What do you want, and why?' - ]) expect(subject.question_count).to eq(3) end context 'when no questions are present' do - let(:stubbed_sentences) { ['This is a statement.', 'Another statement.'] } + subject { 'This is a statement. Another statement.' } it 'returns zero' do expect(subject.question_count).to eq(0) diff --git a/services/QuillLMS/engines/evidence/spec/services/evidence/gen_ai/evaluation/feedback_evaluator_spec.rb b/services/QuillLMS/engines/evidence/spec/services/evidence/gen_ai/evaluation/feedback_evaluator_spec.rb index 6de1b2fcd51..b3ec58300b5 100644 --- a/services/QuillLMS/engines/evidence/spec/services/evidence/gen_ai/evaluation/feedback_evaluator_spec.rb +++ b/services/QuillLMS/engines/evidence/spec/services/evidence/gen_ai/evaluation/feedback_evaluator_spec.rb @@ -5,10 +5,10 @@ describe Evidence::GenAI::Evaluation::FeedbackEvaluator do let(:feedback) { 'This is some feedback text.' } let(:entry) { 'This is the original entry.' } - let(:tone_checker) { class_double(run: tone_result) } - let(:grammar_checker) { class_double(run: grammar_result) } - let(:verbose_checker) { class_double(run: verbose_result) } - let(:direction_count_checker) { class_double(run: direction_count_result) } + let(:tone_checker) { double(run: tone_result) } + let(:grammar_checker) { double(run: grammar_result) } + let(:verbose_checker) { double(run: verbose_result) } + let(:direction_count_checker) { double(run: direction_count_result) } let(:tone_result) { double(optimal: true, errors: []) } let(:grammar_result) { double(optimal: true, errors: []) } From 2d8be046e66931aeb22813e7ad21bb14c7b4384a Mon Sep 17 00:00:00 2001 From: Daniel Drabik Date: Thu, 23 Jan 2025 12:43:33 -0500 Subject: [PATCH 23/51] =?UTF-8?q?Rename=20Scalpel=20to=20better=20name=20s?= =?UTF-8?q?ince=20I=E2=80=99ve=20edited=20it=20a=20bunch=20already.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../{scalpel.rb => sentence_extractor.rb} | 21 +++++----- .../lib/core_ext/string_extensions.rb | 2 +- .../lib/core_ext/string_extensions_spec.rb | 4 +- ...pel_spec.rb => sentence_extractor_spec.rb} | 40 +++++++++---------- 4 files changed, 35 insertions(+), 32 deletions(-) rename services/QuillLMS/engines/evidence/app/services/evidence/{scalpel.rb => sentence_extractor.rb} (89%) rename services/QuillLMS/engines/evidence/spec/services/evidence/{scalpel_spec.rb => sentence_extractor_spec.rb} (78%) diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/scalpel.rb b/services/QuillLMS/engines/evidence/app/services/evidence/sentence_extractor.rb similarity index 89% rename from services/QuillLMS/engines/evidence/app/services/evidence/scalpel.rb rename to services/QuillLMS/engines/evidence/app/services/evidence/sentence_extractor.rb index 1c87e4438f4..29ca092576e 100644 --- a/services/QuillLMS/engines/evidence/app/services/evidence/scalpel.rb +++ b/services/QuillLMS/engines/evidence/app/services/evidence/sentence_extractor.rb @@ -1,23 +1,26 @@ # frozen_string_literal: true # -# File copied (and edited) from a 1 file gem at https://github.com/louismullie/scalpel -# since RubyGems doesn't have the last version and there were some bugs / needed better test coverage +# Based on the Scalpel gem https://github.com/louismullie/scalpel +# Note, RubyGems doesn't have the last version of that gem and there were some bugs / it needed better test coverage +# So started with a copied file and made many edits # # Sentence segmentation based on a set of predefined # rules that handle a large number of usage cases of # sentence enders. The idea is to remove all cases of # .!? being used for other purposes than marking a # full stop before naively segmenting the text. -class Evidence::Scalpel - # Current version. - VERSION = '0.3.0' +class Evidence::SentenceExtractor < Evidence::ApplicationService - # Segment a text using the Scalpel algorithm. - # This will eventually be ported to a gem. - def self.cut(text) + attr_reader :original_text + + def initialize(original_text) + @original_text = original_text + end + + def run # Get a copy of the string. - text = text.dup.to_s + text = original_text.dup.to_s # Remove composite abbreviations. text.gsub!('et al.', 'etal.') # Remove list enders before a comma. diff --git a/services/QuillLMS/engines/evidence/lib/core_ext/string_extensions.rb b/services/QuillLMS/engines/evidence/lib/core_ext/string_extensions.rb index f34f93b6a1c..603baa32231 100644 --- a/services/QuillLMS/engines/evidence/lib/core_ext/string_extensions.rb +++ b/services/QuillLMS/engines/evidence/lib/core_ext/string_extensions.rb @@ -13,7 +13,7 @@ class String ) /ix - def sentences = Evidence::Scalpel.cut(self) + def sentences = Evidence::SentenceExtractor.run(self) def sentence_count = sentences.size def question_sentences = sentences.select { |s| s.match?(QUESTION_PATTERN) } diff --git a/services/QuillLMS/engines/evidence/spec/lib/core_ext/string_extensions_spec.rb b/services/QuillLMS/engines/evidence/spec/lib/core_ext/string_extensions_spec.rb index c278bbd2ac0..f8221a7b94c 100644 --- a/services/QuillLMS/engines/evidence/spec/lib/core_ext/string_extensions_spec.rb +++ b/services/QuillLMS/engines/evidence/spec/lib/core_ext/string_extensions_spec.rb @@ -12,8 +12,8 @@ expect(subject.sentences).to eq(stubbed_sentences) end - it 'calls Scalpel.cut with the input string' do - expect(Evidence::Scalpel).to receive(:cut).with(subject) + it 'calls Evidence::SentenceExtractor.run with the input string' do + expect(Evidence::SentenceExtractor).to receive(:run).with(subject) subject.sentences end diff --git a/services/QuillLMS/engines/evidence/spec/services/evidence/scalpel_spec.rb b/services/QuillLMS/engines/evidence/spec/services/evidence/sentence_extractor_spec.rb similarity index 78% rename from services/QuillLMS/engines/evidence/spec/services/evidence/scalpel_spec.rb rename to services/QuillLMS/engines/evidence/spec/services/evidence/sentence_extractor_spec.rb index 874dbc4e5a8..215bfa2f98a 100644 --- a/services/QuillLMS/engines/evidence/spec/services/evidence/scalpel_spec.rb +++ b/services/QuillLMS/engines/evidence/spec/services/evidence/sentence_extractor_spec.rb @@ -2,115 +2,115 @@ require 'rails_helper' -RSpec.describe Evidence::Scalpel do - describe '.cut' do +RSpec.describe Evidence::SentenceExtractor do + describe '#run' do it 'handles simple sentences correctly' do text = 'This is a sentence. This is another one!' result = ['This is a sentence.', 'This is another one!'] - expect(described_class.cut(text)).to eq(result) + expect(described_class.run(text)).to eq(result) end it 'handles sentence ending abbreviation' do text = 'Dr. Smith went to Washington, etc. It was sunny.' result = ['Dr. Smith went to Washington, etc.', 'It was sunny.'] - expect(described_class.cut(text)).to eq(result) + expect(described_class.run(text)).to eq(result) end it 'handles sentence ending abbreviation with et al.' do text = 'Dr. Smith went to Washington, et al. It was sunny.' result = ['Dr. Smith went to Washington, et al.', 'It was sunny.'] - expect(described_class.cut(text)).to eq(result) + expect(described_class.run(text)).to eq(result) end it 'handles abbreviations correctly' do text = 'Dr. Smith went to Washington. It was sunny.' result = ['Dr. Smith went to Washington.', 'It was sunny.'] - expect(described_class.cut(text)).to eq(result) + expect(described_class.run(text)).to eq(result) end it 'handles e.g. correctly' do text = 'Please bring the necessary supplies, e.g. pens, paper, and books.' result = ['Please bring the necessary supplies, e.g. pens, paper, and books.'] - expect(described_class.cut(text)).to eq(result) + expect(described_class.run(text)).to eq(result) end it 'handles etc. correctly' do text = "Please bring the necessary supplies like pens, paper, etc., and don't forget your notebook." result = ["Please bring the necessary supplies like pens, paper, etc., and don't forget your notebook."] - expect(described_class.cut(text)).to eq(result) + expect(described_class.run(text)).to eq(result) end # Failing test. Might need LLM if we need this. # it 'common test sentence, mid-abbreviations followed by capital and sentence-ending abbreviation followed by capital' do # text = "Hello world. My name is Mr. Smith. I work for the U.S. Government and I live in the U.S. I live in New York." # result = ["Hello world.", "My name is Mr. Smith.", "I work for the U.S. Government and I live in the U.S.", "I live in New York."] - # expect(described_class.cut(text)).to eq(result) + # expect(described_class.run(text)).to eq(result) # end it 'handles etc. as a sentence ender' do text = "Please bring the necessary supplies like pens, paper, etc. Also, don't forget your notebook." result = ['Please bring the necessary supplies like pens, paper, etc.', "Also, don't forget your notebook."] - expect(described_class.cut(text)).to eq(result) + expect(described_class.run(text)).to eq(result) end it 'handles numbers with periods' do text = 'The value is 3.14. That is pi.' result = ['The value is 3.14.', 'That is pi.'] - expect(described_class.cut(text)).to eq(result) + expect(described_class.run(text)).to eq(result) end it 'handles ellipses correctly' do text = 'I was thinking... maybe we should go. What do you think?' result = ['I was thinking...', 'maybe we should go.', 'What do you think?'] - expect(described_class.cut(text)).to eq(result) + expect(described_class.run(text)).to eq(result) end it 'handles titles and initials correctly' do text = 'Mr. J.D. Rockefeller was a famous figure. He lived in the U.S.' result = ['Mr. J.D. Rockefeller was a famous figure.', 'He lived in the U.S.'] - expect(described_class.cut(text)).to eq(result) + expect(described_class.run(text)).to eq(result) end it 'handles nested quotes correctly' do text = "'Did he say, \"Yes\"?' she asked." result = ["'Did he say, \"Yes\"?'", 'she asked.'] - expect(described_class.cut(text)).to eq(result) + expect(described_class.run(text)).to eq(result) end it 'handles punctuation near quotes correctly' do text = "She said, 'Hello.' Then she left." result = ["She said, 'Hello.'", 'Then she left.'] - expect(described_class.cut(text)).to eq(result) + expect(described_class.run(text)).to eq(result) end it 'handles parentheses with sentence enders correctly' do text = 'He was late (again!). She was upset.' result = ['He was late (again!).', 'She was upset.'] - expect(described_class.cut(text)).to eq(result) + expect(described_class.run(text)).to eq(result) end it 'handles parentheses within sentence if it has sentence ender' do text = "He was latte (spelling?), but that isn't important. She was upset." result = ["He was latte (spelling?), but that isn't important.", 'She was upset.'] - expect(described_class.cut(text)).to eq(result) + expect(described_class.run(text)).to eq(result) end it 'handles extra whitespace correctly' do text = ' Hello world. This is a test. ' result = ['Hello world.', 'This is a test.'] - expect(described_class.cut(text)).to eq(result) + expect(described_class.run(text)).to eq(result) end it 'returns an empty array for empty input' do text = '' result = [] - expect(described_class.cut(text)).to eq(result) + expect(described_class.run(text)).to eq(result) end it 'handles floating point numbers without leading zeros' do text = 'The value was .5. It increased to 1.5.' result = ['The value was .5.', 'It increased to 1.5.'] - expect(described_class.cut(text)).to eq(result) + expect(described_class.run(text)).to eq(result) end end end From 0647b7e7298158d9ae6f64f1ca86f5b68b822181 Mon Sep 17 00:00:00 2001 From: Daniel Drabik Date: Thu, 23 Jan 2025 12:46:09 -0500 Subject: [PATCH 24/51] Lint. --- .../evidence/app/services/evidence/sentence_extractor.rb | 1 - .../QuillLMS/engines/evidence/lib/core_ext/string_extensions.rb | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/sentence_extractor.rb b/services/QuillLMS/engines/evidence/app/services/evidence/sentence_extractor.rb index 29ca092576e..b7dc222f63d 100644 --- a/services/QuillLMS/engines/evidence/app/services/evidence/sentence_extractor.rb +++ b/services/QuillLMS/engines/evidence/app/services/evidence/sentence_extractor.rb @@ -11,7 +11,6 @@ # .!? being used for other purposes than marking a # full stop before naively segmenting the text. class Evidence::SentenceExtractor < Evidence::ApplicationService - attr_reader :original_text def initialize(original_text) diff --git a/services/QuillLMS/engines/evidence/lib/core_ext/string_extensions.rb b/services/QuillLMS/engines/evidence/lib/core_ext/string_extensions.rb index 603baa32231..640584fa003 100644 --- a/services/QuillLMS/engines/evidence/lib/core_ext/string_extensions.rb +++ b/services/QuillLMS/engines/evidence/lib/core_ext/string_extensions.rb @@ -16,7 +16,7 @@ class String def sentences = Evidence::SentenceExtractor.run(self) def sentence_count = sentences.size - def question_sentences = sentences.select { |s| s.match?(QUESTION_PATTERN) } + def question_sentences = sentences.grep(QUESTION_PATTERN) def question_count question_sentences.reduce(0) do |count, sentence| From 26e5123c32a9158e77fbe4672fb35951a6b49625 Mon Sep 17 00:00:00 2001 From: Daniel Drabik Date: Thu, 23 Jan 2025 13:32:16 -0500 Subject: [PATCH 25/51] =?UTF-8?q?Don=E2=80=99t=20modify=20String=20from=20?= =?UTF-8?q?an=20Engine=20(rethinking=20this=20is=20bad=20form).?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../gen_ai/evaluation/base_checker.rb | 1 + .../evaluation/direction_count/checker.rb | 4 +- .../evaluation/misacknowledgement/checker.rb | 30 ---------- .../misacknowledgement/prompt_builder.rb | 57 ------------------- .../prompts/2025_01_15_initial.md | 12 ---- .../prompts/2025_01_15_rephrase.md | 18 ------ .../prompts/2025_01_17_explain_feedback.md | 21 ------- .../prompts/2025_01_17_info_in_entry.md | 13 ----- .../prompts/2025_01_17_iteration.md | 11 ---- .../prompts/2025_01_17_iteration_no_reason.md | 8 --- .../2025_01_17_iteration_with_examples.md | 13 ----- .../gen_ai/evaluation/verbose/checker.rb | 8 ++- .../services/evidence/question_counter.rb} | 26 +++++---- .../evidence/lib/core_ext/extensions.rb | 3 - .../engines/evidence/lib/evidence/engine.rb | 4 -- .../lib/core_ext/string_extensions_spec.rb | 57 ------------------- .../evidence/question_counter_spec.rb | 29 ++++++++++ 17 files changed, 55 insertions(+), 260 deletions(-) delete mode 100644 services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/checker.rb delete mode 100644 services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompt_builder.rb delete mode 100644 services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompts/2025_01_15_initial.md delete mode 100644 services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompts/2025_01_15_rephrase.md delete mode 100644 services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompts/2025_01_17_explain_feedback.md delete mode 100644 services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompts/2025_01_17_info_in_entry.md delete mode 100644 services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompts/2025_01_17_iteration.md delete mode 100644 services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompts/2025_01_17_iteration_no_reason.md delete mode 100644 services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompts/2025_01_17_iteration_with_examples.md rename services/QuillLMS/engines/evidence/{lib/core_ext/string_extensions.rb => app/services/evidence/question_counter.rb} (52%) delete mode 100644 services/QuillLMS/engines/evidence/lib/core_ext/extensions.rb delete mode 100644 services/QuillLMS/engines/evidence/spec/lib/core_ext/string_extensions_spec.rb create mode 100644 services/QuillLMS/engines/evidence/spec/services/evidence/question_counter_spec.rb diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/base_checker.rb b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/base_checker.rb index cd4261f61e1..b13235607fd 100644 --- a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/base_checker.rb +++ b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/base_checker.rb @@ -18,6 +18,7 @@ def run end private def optimal = Result.new(optimal: true, errors: []) + private def feedback_sentences = Evidence::SentenceExtractor.run(feedback) end end end diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/direction_count/checker.rb b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/direction_count/checker.rb index 5bf984dd8a8..24a1ab04101 100644 --- a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/direction_count/checker.rb +++ b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/direction_count/checker.rb @@ -6,11 +6,13 @@ module Evaluation module DirectionCount class Checker < Evaluation::BaseChecker def run - return suboptimal if feedback.question_count > 1 + return suboptimal if question_count > 1 optimal end + + private def question_count = Evidence::QuestionCounter.run(feedback) private def suboptimal = Result.new(optimal: false, errors: ['two_directions']) end end diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/checker.rb b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/checker.rb deleted file mode 100644 index 19a57256b7d..00000000000 --- a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/checker.rb +++ /dev/null @@ -1,30 +0,0 @@ -# frozen_string_literal: true - -module Evidence - module GenAI - module Evaluation - module Misacknowledgement - class Checker < Evaluation::BaseChecker - include Evidence::LlmRequestable - - CLEAR_RESPONSE_REGEX = /clearing your response/ - - def run - optimal_llm_response? ? optimal : suboptimal - end - - private def suboptimal = Result.new(optimal: false, errors: ['misacknowledgement']) - private def delimiter = PromptBuilder::DELIMITER - - private def llm_entry = [entry, delimiter, acknowledgement].join - private def system_prompt = Evidence::GenAI::Evaluation::Misacknowledgement::PromptBuilder.run(entry:) - private def model = CHAT_API::LARGE_MODEL - - private def acknowledgement - feedback.sentences.find { |s| !s.match?(CLEAR_RESPONSE_REGEX) } - end - end - end - end - end -end diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompt_builder.rb b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompt_builder.rb deleted file mode 100644 index 9882c9f8f50..00000000000 --- a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompt_builder.rb +++ /dev/null @@ -1,57 +0,0 @@ -# frozen_string_literal: true - -module Evidence - module GenAI - module Evaluation - module Misacknowledgement - class PromptBuilder < ApplicationService - include Evidence::MarkdownFormatter - - TEMPLATE_FOLDER = 'app/services/evidence/gen_ai/evaluation/misacknowledgement/prompts/' - TEMPLATE = '2025_01_17_iteration_with_examples.md' - DELIMITER = ' :: ' - - Example = Data.define(:entry, :feedback, :optimal) - - EXAMPLES = [ - Example.new('the ocean is deep', "It's true that the ocean is deep.", true), - Example.new('the ocean is deep', 'Instead of explaining that the ocean is deep, write about the issue with ocean pollution.', true), - Example.new('the war was long and hard', "It's true, the war was long.", true), - Example.new('they went to the ballgame', "It's true that Tom and his friends went to the ballgame.", true), - Example.new('there were many people involved in the battle', "It's true that the battle involved many people.", true), - Example.new('they wanted a manned mission to space', 'Instead of writing about NASA wanted a manned mission to space, focus on the budget issues in the text.', true), - Example.new('the war was long and hard', "It's true war was long.", true), - Example.new('the pasta is made with wheat flour and water', "It's true that pasta has many ingredients, but instead write about the cooking method.", true), - Example.new('the pasta is made with wheat flour and water', 'Instead of talking about where pasta was invented, write about the cooking method.', false), - Example.new('the boys were scared.', "It's true that the boys were small and frightened.", false), - Example.new('some gibberish.', "It's true pizza is delicious.", false), - Example.new('the pasta is made with wheat flour and water', 'Instead of talking about where pasta was invented, write about the cooking method.', false) - ] - - attr_reader :entry - - def initialize(entry:) - @entry = entry - end - - private def template_variables - { - examples_table:, - delimiter: - } - end - - private def delimiter = DELIMITER - - private def examples_table = markdown_table_rows(examples) - private def examples = EXAMPLES.map { |e| [[e.entry, delimiter, e.feedback].join, e.optimal] } - - def run = template % template_variables - - private def template = File.read(template_file_path) - private def template_file_path = Evidence::Engine.root.join(TEMPLATE_FOLDER, TEMPLATE) - end - end - end - end -end diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompts/2025_01_15_initial.md b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompts/2025_01_15_initial.md deleted file mode 100644 index 1561e2b5a19..00000000000 --- a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompts/2025_01_15_initial.md +++ /dev/null @@ -1,12 +0,0 @@ -Evaluate whether the chatbot's response acknowledges some of the information in the user's input. Paraphrasing, rewording, or reorganizing the information is acceptable. - -Based on your evaluation, return a JSON response: - -- {"optimal": true} if the chatbot's response accurately reflects the user's input. -- {"optimal": false} if the chatbot's response introduces any unsupported information that isn't in the user's input. - -User Input: - -%s - -Chatbot Response (entry): diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompts/2025_01_15_rephrase.md b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompts/2025_01_15_rephrase.md deleted file mode 100644 index 0dc11897bdf..00000000000 --- a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompts/2025_01_15_rephrase.md +++ /dev/null @@ -1,18 +0,0 @@ -You are an expert feedback evaluator. You will analyze a piece of feedback given and determine if it meets the feedback guidelines. Use the student's answer for reference. - -Student's Answer -``` -%s -``` - -Go through these rules one by one and evaluate the feedback. If the feedback does not meet the criteria of a rule, return JSON {"optimal" : false}. Stop evaluating rule if you hit a failing rule. If the feedback passes all of the rules, return {"optimal" : true} - -## Examples -|user_input|chatbot_response|optimal| -|----|----|----| -%s - -### Rules -1. Return JSON {'optimal' : true} - -Feedback: diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompts/2025_01_17_explain_feedback.md b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompts/2025_01_17_explain_feedback.md deleted file mode 100644 index 72447b67f9e..00000000000 --- a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompts/2025_01_17_explain_feedback.md +++ /dev/null @@ -1,21 +0,0 @@ -You are a chatbot response evaluator. - -Based on your evaluation, return a JSON response with three keys 'optimal', 'reason', and 'acknowledgement'. - -- {"optimal": true} if the chatbot's response accurately reflects the user's input without introducing unsupported details, assumptions, or altering the meaning. -- {"optimal": false} if the chatbot's response does not reflect the information in the user input. - -The 'reason' key should be text describing why you chose optimal true or false. Please give a 'reason' in all cases, even if 'optimal' is true. - -The entry will be of the form "user_input%schatbot_response". - -Steps -1. Separate the user_input from the chatbot_response -2. Determine the acknowledgement, the subsection of the chatbot_response that acknowledges what the user wrote. -3. Compare the acknowledgement and what the user wrote, and determine if the acknowledgement generally reflects what the user wrote -4. Return the acknowledgement, optimal, and reason keys. - -## Examples -|entry|optimal| -|----|----| -%s diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompts/2025_01_17_info_in_entry.md b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompts/2025_01_17_info_in_entry.md deleted file mode 100644 index 1224fdd362b..00000000000 --- a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompts/2025_01_17_info_in_entry.md +++ /dev/null @@ -1,13 +0,0 @@ -Evaluate whether the chatbot's response introduces any new information, assumptions, or changes in meaning that are not explicitly stated or implied in the user's input. Paraphrasing is acceptable. You can also ignore transitional or conversational words such as "Instead of writing" or "That's true" and focus only on the facts presented. - -Based on your evaluation, return a JSON response with two keys 'optimal' and 'reason'. - -- {"optimal": true} if the chatbot's response accurately reflects the user's input without introducing unsupported details, assumptions, or altering the meaning. -- {"optimal": false} if the chatbot's response does not reflect the information in the user input. - -The 'reason' key should be text describing why you chose optimal true or false. Please give a 'reason' in all cases, even if 'optimal' is true. - -Be strict in identifying inaccuracies, but do not flag stylistic differences or paraphrasing that maintains the original meaning. Return only the JSON output. - -The entry will be of the form "user_input%schatbot_response". - diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompts/2025_01_17_iteration.md b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompts/2025_01_17_iteration.md deleted file mode 100644 index c74612ce2ae..00000000000 --- a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompts/2025_01_17_iteration.md +++ /dev/null @@ -1,11 +0,0 @@ -You are an expert feedback evaluator. The feedback that you are analyzing should acknowledge what the user wrote, and give further instructions. You are to ignore the feedback's further instructions and just analyze the feedback's acknowledgement of what the user wrote, e.g. ignore the instructions in the feedback such as, "Instead of writing", "That's true", "focus on this", "write about this.". - -Based on your evaluation, return a JSON response with two keys 'optimal' and 'reason'. - -- {"optimal": true} if the feedback accurately acknowledges the user input. Paraphrasing and rewording is ok. The acknowledgement does not need to contain all the information in the user input. Default to labeling {"optimal": true} unless you have a clear reason not to. -- {"optimal": false} if the acknowledgement in the feedback does not reflect the information in the user input. - -The 'reason' key should be text describing why you chose optimal true or false. Please give a 'reason' in all cases, even if 'optimal' is true. - -The entry will be of the form "user_input%schatbot_response". - diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompts/2025_01_17_iteration_no_reason.md b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompts/2025_01_17_iteration_no_reason.md deleted file mode 100644 index 0b722906fd9..00000000000 --- a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompts/2025_01_17_iteration_no_reason.md +++ /dev/null @@ -1,8 +0,0 @@ -You are an expert feedback evaluator. The feedback that you are analyzing should acknowledge what the user wrote, and give further instructions. You are to ignore the feedback's further instructions and just analyze the feedback's acknowledgement of what the user wrote, e.g. ignore the instructions in the feedback such as, "Instead of writing", "That's true", "focus on this", "write about this.". You can also assume that pronouns in the user_input reference proper nouns or nouns in the feedback (or vice versa). - -Based on your evaluation, return a JSON response with one key 'optimal'. - -- {"optimal": true} if the feedback accurately acknowledges the user input. Paraphrasing and rewording is ok. The acknowledgement does not need to contain all the information in the user input. Default to labeling {"optimal": true} unless you have a clear reason not to. -- {"optimal": false} if the acknowledgement in the feedback does not reflect the information in the user input. - -The entry will be of the form "user_input%schatbot_response". diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompts/2025_01_17_iteration_with_examples.md b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompts/2025_01_17_iteration_with_examples.md deleted file mode 100644 index 4d7c6ce228e..00000000000 --- a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/misacknowledgement/prompts/2025_01_17_iteration_with_examples.md +++ /dev/null @@ -1,13 +0,0 @@ -You are an expert feedback evaluator. The feedback that you are analyzing should acknowledge what the user wrote, and give further instructions. You are to ignore the feedback's further instructions and just analyze the feedback's acknowledgement of what the user wrote, e.g. ignore the instructions in the feedback such as, "Instead of writing", "That's true", "focus on this", "write about this.". You can also assume that pronouns in the user_input reference proper nouns or nouns in the feedback (or vice versa). - -Based on your evaluation, return a JSON response with one key 'optimal'. - -- {"optimal": true} if the feedback accurately acknowledges the user input. Paraphrasing and rewording is ok. The acknowledgement does not need to contain all the information in the user input. Default to labeling {"optimal": true} unless you have a clear reason not to. -- {"optimal": false} if the acknowledgement in the feedback does not reflect the information in the user input. - -The entry will be of the form "user_input%schatbot_response". - -## Examples -|entry|optimal| -|----|----| -%s diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/verbose/checker.rb b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/verbose/checker.rb index 26cd7920ffd..a902ddce94a 100644 --- a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/verbose/checker.rb +++ b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/verbose/checker.rb @@ -15,12 +15,16 @@ def run end private def too_long? - return true if feedback.sentence_count > MAX_SENTENCES + return true if feedback_sentence_count > MAX_SENTENCES - feedback.sentence_count == MAX_SENTENCES && + feedback_sentence_count == MAX_SENTENCES && feedback.size > MAX_SIZE end + private def feedback_sentence_count + @feedback_sentence_count ||= feedback_sentences.size + end + private def suboptimal = Result.new(optimal: false, errors: ['verbose']) end end diff --git a/services/QuillLMS/engines/evidence/lib/core_ext/string_extensions.rb b/services/QuillLMS/engines/evidence/app/services/evidence/question_counter.rb similarity index 52% rename from services/QuillLMS/engines/evidence/lib/core_ext/string_extensions.rb rename to services/QuillLMS/engines/evidence/app/services/evidence/question_counter.rb index 640584fa003..e8776222747 100644 --- a/services/QuillLMS/engines/evidence/lib/core_ext/string_extensions.rb +++ b/services/QuillLMS/engines/evidence/app/services/evidence/question_counter.rb @@ -1,8 +1,9 @@ # frozen_string_literal: true -class String - QUESTION_PATTERN = /\?\s|[^\?]+\?$/ - MULTI_QUESTION_PATTERN = / +module Evidence + class QuestionCounter < ApplicationService + QUESTION_PATTERN = /\?\s|[^\?]+\?$/ + MULTI_QUESTION_PATTERN = / ( \b(?:who|what|when|where|why|how|which|whose)\b # A question word .*? # Any text after the question word @@ -13,15 +14,20 @@ class String ) /ix - def sentences = Evidence::SentenceExtractor.run(self) - def sentence_count = sentences.size + attr_reader :text - def question_sentences = sentences.grep(QUESTION_PATTERN) + def initialize(text) + @text = text + end - def question_count - question_sentences.reduce(0) do |count, sentence| - questions_in_sentence = sentence.match?(MULTI_QUESTION_PATTERN) ? 2 : 1 - count + questions_in_sentence + def run + question_sentences.reduce(0) do |count, sentence| + questions_in_sentence = sentence.match?(MULTI_QUESTION_PATTERN) ? 2 : 1 + count + questions_in_sentence + end end + + private def sentences = SentenceExtractor.run(text) + private def question_sentences = sentences.grep(QUESTION_PATTERN) end end diff --git a/services/QuillLMS/engines/evidence/lib/core_ext/extensions.rb b/services/QuillLMS/engines/evidence/lib/core_ext/extensions.rb deleted file mode 100644 index fd7d225715c..00000000000 --- a/services/QuillLMS/engines/evidence/lib/core_ext/extensions.rb +++ /dev/null @@ -1,3 +0,0 @@ -# frozen_string_literal: true - -require_relative 'string_extensions' diff --git a/services/QuillLMS/engines/evidence/lib/evidence/engine.rb b/services/QuillLMS/engines/evidence/lib/evidence/engine.rb index c18b967ad89..2b49e46f44c 100644 --- a/services/QuillLMS/engines/evidence/lib/evidence/engine.rb +++ b/services/QuillLMS/engines/evidence/lib/evidence/engine.rb @@ -28,10 +28,6 @@ class Engine < ::Rails::Engine g.template_engine nil end - initializer 'evidence.load_core_extensions' do - require_relative '../core_ext/extensions' - end - if defined?(FactoryBot) initializer 'evidence.factories', after: 'factory_bot.set_factory_paths' do FactoryBot.definition_file_paths << File.expand_path('../../../spec/factories/evidence', __FILE__) diff --git a/services/QuillLMS/engines/evidence/spec/lib/core_ext/string_extensions_spec.rb b/services/QuillLMS/engines/evidence/spec/lib/core_ext/string_extensions_spec.rb deleted file mode 100644 index f8221a7b94c..00000000000 --- a/services/QuillLMS/engines/evidence/spec/lib/core_ext/string_extensions_spec.rb +++ /dev/null @@ -1,57 +0,0 @@ -# frozen_string_literal: true - -require 'rails_helper' - -describe String do - let(:stubbed_sentences) { ['Who are you?', 'What do you want?'] } - - subject { 'Who are you? What do you want?' } - - describe '#sentences' do - it 'returns an array of sentences using the Scalpel.cut method' do - expect(subject.sentences).to eq(stubbed_sentences) - end - - it 'calls Evidence::SentenceExtractor.run with the input string' do - expect(Evidence::SentenceExtractor).to receive(:run).with(subject) - - subject.sentences - end - end - - describe '#sentence_count' do - it 'returns the number of sentences' do - expect(subject.sentence_count).to eq(stubbed_sentences.size) - end - end - - describe '#question_sentences' do - it 'returns sentences that match the QUESTION_PATTERN' do - expect(subject.question_sentences).to eq(stubbed_sentences) - end - - context 'when a sentence does not contain a question' do - subject { 'This is not a question. What do you mean?' } - - it 'filters out non-question sentences' do - expect(subject.question_sentences).to eq(['What do you mean?']) - end - end - end - - describe '#question_count' do - subject { 'Who are you? What do you want, and why?' } - - it 'returns the total count of questions, including multi-question sentences' do - expect(subject.question_count).to eq(3) - end - - context 'when no questions are present' do - subject { 'This is a statement. Another statement.' } - - it 'returns zero' do - expect(subject.question_count).to eq(0) - end - end - end -end diff --git a/services/QuillLMS/engines/evidence/spec/services/evidence/question_counter_spec.rb b/services/QuillLMS/engines/evidence/spec/services/evidence/question_counter_spec.rb new file mode 100644 index 00000000000..649c97552a6 --- /dev/null +++ b/services/QuillLMS/engines/evidence/spec/services/evidence/question_counter_spec.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +require 'rails_helper' + +describe Evidence::QuestionCounter do + let(:text) { 'Who are you? What do you want?' } + + subject { described_class.run(text) } + + describe '#run' do + context 'one question' do + let(:text) { 'Who are you? This is a statement.' } + + it { expect(subject).to eq(1) } + end + + context 'no questions' do + let(:text) { 'This is a paragraph of statements. Statement 2. Another one!' } + + it { expect(subject).to eq(0) } + end + + context 'multiple questions in a sentence' do + let(:text) { 'Who are you? What do you want, and why?' } + + it { expect(subject).to eq(3) } + end + end +end From 9fbbe436feb4ed3841d5ee55edc97793edc47e3f Mon Sep 17 00:00:00 2001 From: Daniel Drabik Date: Thu, 23 Jan 2025 13:41:05 -0500 Subject: [PATCH 26/51] Lint. --- .../evidence/gen_ai/evaluation/direction_count/checker.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/direction_count/checker.rb b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/direction_count/checker.rb index 24a1ab04101..ab843f7a62e 100644 --- a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/direction_count/checker.rb +++ b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/direction_count/checker.rb @@ -11,7 +11,6 @@ def run optimal end - private def question_count = Evidence::QuestionCounter.run(feedback) private def suboptimal = Result.new(optimal: false, errors: ['two_directions']) end From fae954f79c210976176683bb256c4c97fc264806 Mon Sep 17 00:00:00 2001 From: Daniel Drabik Date: Thu, 23 Jan 2025 13:50:05 -0500 Subject: [PATCH 27/51] Fix script, delete unused file, lint. --- .../concerns/evidence/llm_requestable.rb | 25 ------------------- .../gen_ai/evaluation/data_fetcher.rb | 2 +- .../QuillLMS/lib/tasks/gen_ai_evaluation.thor | 2 +- 3 files changed, 2 insertions(+), 27 deletions(-) delete mode 100644 services/QuillLMS/engines/evidence/app/services/concerns/evidence/llm_requestable.rb diff --git a/services/QuillLMS/engines/evidence/app/services/concerns/evidence/llm_requestable.rb b/services/QuillLMS/engines/evidence/app/services/concerns/evidence/llm_requestable.rb deleted file mode 100644 index 4b22a9aa053..00000000000 --- a/services/QuillLMS/engines/evidence/app/services/concerns/evidence/llm_requestable.rb +++ /dev/null @@ -1,25 +0,0 @@ -# frozen_string_literal: true - -module Evidence - # TODO: Get inflection to work properly. - module LlmRequestable - extend ActiveSupport::Concern - - CHAT_API = Evidence::Gemini::Chat - OPTIMAL_KEY = 'optimal' - - # default to true for malformed responses, e.g. nil, missing key - private def optimal_llm_response? - llm_response.fetch(OPTIMAL_KEY, true) - rescue NoMethodError => e - Evidence.error_notifier.report(e) - true - end - - private def model = CHAT_API::SMALL_MODEL - - private def llm_response - @llm_response ||= CHAT_API.run(system_prompt:, entry: llm_entry, model:) - end - end -end diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/data_fetcher.rb b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/data_fetcher.rb index e06606cf701..118d095f24a 100644 --- a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/data_fetcher.rb +++ b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/data_fetcher.rb @@ -11,7 +11,7 @@ class DataFetcher < ApplicationService SOLDIERS_BECAUSE = 'underage_soldiers_because.csv' HAITIAN_BECAUSE = 'new_haitian_because.csv' - DEFAULT_FILE = HAITIAN_BECAUSE + DEFAULT_FILE = WHALES_SO CSV_FILE_PATH = "#{Evidence::Engine.root}/app/services/evidence/gen_ai/evaluation/data/%s" attr_reader :rule, :optimal diff --git a/services/QuillLMS/lib/tasks/gen_ai_evaluation.thor b/services/QuillLMS/lib/tasks/gen_ai_evaluation.thor index 8383824d407..63b28f5bb0f 100644 --- a/services/QuillLMS/lib/tasks/gen_ai_evaluation.thor +++ b/services/QuillLMS/lib/tasks/gen_ai_evaluation.thor @@ -37,7 +37,7 @@ class GenAIEvaluation < Thor feedback = item.feedback rule = item.rule entry = item.entry - acknowledgement = feedback.sentences.find { |s| !s.match?(CLEAR_RESPONSE_REGEX) } + acknowledgement = Evidence::SentenceExtractor.run(feedback).find { |s| !s.match?(CLEAR_RESPONSE_REGEX) } response = Evidence::GenAI::Evaluation::FeedbackEvaluator.run(feedback:, entry:) optimal_output = response.optimal From 574f4b5a48defa04c80da47f8edbe65cd399a6e2 Mon Sep 17 00:00:00 2001 From: Daniel Drabik Date: Thu, 23 Jan 2025 13:53:56 -0500 Subject: [PATCH 28/51] Code Cleanup. --- .../gen_ai/evaluation/data_fetcher.rb | 1 - .../services/evidence/sentence_extractor.rb | 160 +++++++++--------- .../evidence/config/initializers/zeitwerk.rb | 1 - .../engines/evidence/lib/evidence/engine.rb | 1 - 4 files changed, 81 insertions(+), 82 deletions(-) diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/data_fetcher.rb b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/data_fetcher.rb index 118d095f24a..de93beab2b2 100644 --- a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/data_fetcher.rb +++ b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/data_fetcher.rb @@ -39,7 +39,6 @@ def run Item.new( entry: row['entry'], feedback: row['feedback'], - # optimal_broad: ActiveModel::Type::Boolean.new.cast(row['optimal_broad']), optimal: ActiveModel::Type::Boolean.new.cast(row['optimal']), rule: row['rule'] ) diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/sentence_extractor.rb b/services/QuillLMS/engines/evidence/app/services/evidence/sentence_extractor.rb index b7dc222f63d..b77ddd47fb0 100644 --- a/services/QuillLMS/engines/evidence/app/services/evidence/sentence_extractor.rb +++ b/services/QuillLMS/engines/evidence/app/services/evidence/sentence_extractor.rb @@ -10,90 +10,92 @@ # sentence enders. The idea is to remove all cases of # .!? being used for other purposes than marking a # full stop before naively segmenting the text. -class Evidence::SentenceExtractor < Evidence::ApplicationService - attr_reader :original_text +module Evidence + class SentenceExtractor < ApplicationService + attr_reader :original_text - def initialize(original_text) - @original_text = original_text - end + def initialize(original_text) + @original_text = original_text + end - def run - # Get a copy of the string. - text = original_text.dup.to_s - # Remove composite abbreviations. - text.gsub!('et al.', 'etal.') - # Remove list enders before a comma. - text.gsub!(/(etc|etal)(\.)(,)/) { "#{::Regexp.last_match(1)}&|&#{::Regexp.last_match(3)}" } - # Remove suspension points. - text.gsub!('...', '&;&.') - # Remove floating point numbers. - text.gsub!(/([0-9]+)\.([0-9]+)/) { "#{::Regexp.last_match(1)}&@&#{::Regexp.last_match(2)}" } - # Handle floats without leading zero. - text.gsub!(/\s\.([0-9]+)/) { " &#&#{::Regexp.last_match(1)}" } - # Remove abbreviations. - text.gsub!(/(?:[A-Za-z]\.){2,}/) { |abbr| abbr.gsub('.', '&-&') } - # Remove initials. - text.gsub!(/(?:[A-Z]\.)/) { |abbr| abbr.gsub('.', '&£&') } - # Remove titles. - text.gsub!(/[A-Z][a-z]{1,2}\./) { |title| title.gsub('.', '&*&') } - # Unstick sentences from each other. - text.gsub!(/([^.?!]\.|!|\?)([^\s"'])/) { "#{::Regexp.last_match(1)} #{::Regexp.last_match(2)}" } - # Remove sentence enders next to quotes. - text.gsub!(/'([.?!])\s?"/) { "&^&#{::Regexp.last_match(1)}" } - text.gsub!(/'([.?!])\s?”/) { "&*&#{::Regexp.last_match(1)}" } - text.gsub!(/([.?!])\s?”/) { "&=&#{::Regexp.last_match(1)}" } - text.gsub!(/([.?!])\s?'"/) { "&,&#{::Regexp.last_match(1)}" } - text.gsub!(/([.?!])\s?'/) { "&%&#{::Regexp.last_match(1)}" } - text.gsub!(/([.?!])\s?"/) { "&$&#{::Regexp.last_match(1)}" } - # Remove sentence enders before parens. - text.gsub!(/([.])\s?(\))/) { "&€1&#{::Regexp.last_match(2)}" } - text.gsub!(/([?])\s?(\))/) { "&€2&#{::Regexp.last_match(2)}" } - text.gsub!(/([!])\s?(\))/) { "&€3&#{::Regexp.last_match(2)}" } + def run + # Get a copy of the string. + text = original_text.dup.to_s + # Remove composite abbreviations. + text.gsub!('et al.', 'etal.') + # Remove list enders before a comma. + text.gsub!(/(etc|etal)(\.)(,)/) { "#{::Regexp.last_match(1)}&|&#{::Regexp.last_match(3)}" } + # Remove suspension points. + text.gsub!('...', '&;&.') + # Remove floating point numbers. + text.gsub!(/([0-9]+)\.([0-9]+)/) { "#{::Regexp.last_match(1)}&@&#{::Regexp.last_match(2)}" } + # Handle floats without leading zero. + text.gsub!(/\s\.([0-9]+)/) { " &#&#{::Regexp.last_match(1)}" } + # Remove abbreviations. + text.gsub!(/(?:[A-Za-z]\.){2,}/) { |abbr| abbr.gsub('.', '&-&') } + # Remove initials. + text.gsub!(/(?:[A-Z]\.)/) { |abbr| abbr.gsub('.', '&£&') } + # Remove titles. + text.gsub!(/[A-Z][a-z]{1,2}\./) { |title| title.gsub('.', '&*&') } + # Unstick sentences from each other. + text.gsub!(/([^.?!]\.|!|\?)([^\s"'])/) { "#{::Regexp.last_match(1)} #{::Regexp.last_match(2)}" } + # Remove sentence enders next to quotes. + text.gsub!(/'([.?!])\s?"/) { "&^&#{::Regexp.last_match(1)}" } + text.gsub!(/'([.?!])\s?”/) { "&*&#{::Regexp.last_match(1)}" } + text.gsub!(/([.?!])\s?”/) { "&=&#{::Regexp.last_match(1)}" } + text.gsub!(/([.?!])\s?'"/) { "&,&#{::Regexp.last_match(1)}" } + text.gsub!(/([.?!])\s?'/) { "&%&#{::Regexp.last_match(1)}" } + text.gsub!(/([.?!])\s?"/) { "&$&#{::Regexp.last_match(1)}" } + # Remove sentence enders before parens. + text.gsub!(/([.])\s?(\))/) { "&€1&#{::Regexp.last_match(2)}" } + text.gsub!(/([?])\s?(\))/) { "&€2&#{::Regexp.last_match(2)}" } + text.gsub!(/([!])\s?(\))/) { "&€3&#{::Regexp.last_match(2)}" } - # Split on any sentence ender. - sentences = text.split(/([.!?])/) - new_sents = [] - # Join the obtained slices. - sentences.each_slice(2) do |slice| - new_sents << slice.join - end - # Repair the damage we've done. - results = [] - new_sents.each do |sentence| - # Skip whitespace zones. - next if sentence.strip == '' + # Split on any sentence ender. + sentences = text.split(/([.!?])/) + new_sents = [] + # Join the obtained slices. + sentences.each_slice(2) do |slice| + new_sents << slice.join + end + # Repair the damage we've done. + results = [] + new_sents.each do |sentence| + # Skip whitespace zones. + next if sentence.strip == '' - # Repair composite abbreviations. - sentence.gsub!('etal', 'et al') - # Repair list enders before a comma. - sentence.gsub!("&\|&", '.') - # Repair initials. - sentence.gsub!('&£&', '.') - # Repair abbreviations. - sentence.gsub!('&-&', '.') - # Repair titles. - sentence.gsub!('&*&', '.') - # Repair suspension points. - sentence.gsub!('&;&.', '...') - # Repair floats. - sentence.gsub!(/([0-9]+)&@&([0-9]+)/) { "#{::Regexp.last_match(1)}.#{::Regexp.last_match(2)}" } - # Repair quotes with sentence enders - sentence.gsub!(/&=&([.!?])/) { "#{::Regexp.last_match(1)}”" } - sentence.gsub!(/&,&([.!?])/) { "#{::Regexp.last_match(1)}'\"" } - sentence.gsub!(/&%&([.!?])/) { "#{::Regexp.last_match(1)}'" } - sentence.gsub!(/&\^&([.?!])/) { "'#{::Regexp.last_match(1)}\"" } - sentence.gsub!(/&\*&([.?!])/) { "'#{::Regexp.last_match(1)}”" } - sentence.gsub!(/&\$&([.!?])/) { "#{::Regexp.last_match(1)}\"" } - # Repair parens with sentence enders - sentence.gsub!('&€1&', '.') - sentence.gsub!('&€2&', '?') - sentence.gsub!('&€3&', '!') + # Repair composite abbreviations. + sentence.gsub!('etal', 'et al') + # Repair list enders before a comma. + sentence.gsub!("&\|&", '.') + # Repair initials. + sentence.gsub!('&£&', '.') + # Repair abbreviations. + sentence.gsub!('&-&', '.') + # Repair titles. + sentence.gsub!('&*&', '.') + # Repair suspension points. + sentence.gsub!('&;&.', '...') + # Repair floats. + sentence.gsub!(/([0-9]+)&@&([0-9]+)/) { "#{::Regexp.last_match(1)}.#{::Regexp.last_match(2)}" } + # Repair quotes with sentence enders + sentence.gsub!(/&=&([.!?])/) { "#{::Regexp.last_match(1)}”" } + sentence.gsub!(/&,&([.!?])/) { "#{::Regexp.last_match(1)}'\"" } + sentence.gsub!(/&%&([.!?])/) { "#{::Regexp.last_match(1)}'" } + sentence.gsub!(/&\^&([.?!])/) { "'#{::Regexp.last_match(1)}\"" } + sentence.gsub!(/&\*&([.?!])/) { "'#{::Regexp.last_match(1)}”" } + sentence.gsub!(/&\$&([.!?])/) { "#{::Regexp.last_match(1)}\"" } + # Repair parens with sentence enders + sentence.gsub!('&€1&', '.') + sentence.gsub!('&€2&', '?') + sentence.gsub!('&€3&', '!') - # Repair floats without leading zeros. - sentence.gsub!(/&#&([0-9]+)/) { ".#{::Regexp.last_match(1)}" } - results << sentence.strip - end + # Repair floats without leading zeros. + sentence.gsub!(/&#&([0-9]+)/) { ".#{::Regexp.last_match(1)}" } + results << sentence.strip + end - results + results + end end end diff --git a/services/QuillLMS/engines/evidence/config/initializers/zeitwerk.rb b/services/QuillLMS/engines/evidence/config/initializers/zeitwerk.rb index ba911d79bca..18cab5bb936 100644 --- a/services/QuillLMS/engines/evidence/config/initializers/zeitwerk.rb +++ b/services/QuillLMS/engines/evidence/config/initializers/zeitwerk.rb @@ -25,7 +25,6 @@ 'llm_prompt_relevant_text' => 'LLMPromptRelevantText', 'llm_prompt_template' => 'LLMPromptTemplate', 'llm_prompt_templates_controller' => 'LLMPromptTemplatesController', - 'llm_requestable' => 'LLMRequestable', 'llms' => 'LLMs', 'llms_controller' => 'LLMsController', 'malformed_json_fixer' => 'MalformedJSONFixer', diff --git a/services/QuillLMS/engines/evidence/lib/evidence/engine.rb b/services/QuillLMS/engines/evidence/lib/evidence/engine.rb index 2b49e46f44c..c3f41ddfa1c 100644 --- a/services/QuillLMS/engines/evidence/lib/evidence/engine.rb +++ b/services/QuillLMS/engines/evidence/lib/evidence/engine.rb @@ -13,7 +13,6 @@ module Evidence class Engine < ::Rails::Engine config.eager_load_paths << "#{config.root}/lib" - isolate_namespace Evidence config.generators do |g| From 1d9e902e20caae4ad7da8073773476d403041d3a Mon Sep 17 00:00:00 2001 From: Daniel Drabik Date: Thu, 30 Jan 2025 14:39:50 -0500 Subject: [PATCH 29/51] Initial working version of Feedback Evaluation. --- ...ch_gen_ai_feedback_evaluations.evidence.rb | 17 +++ services/QuillLMS/db/structure.sql | 104 +++++++++++++++++- .../QuillLMS/engines/evidence/README.rdoc | 4 +- .../research/gen_ai/feedback_evaluation.rb | 37 +++++++ .../models/evidence/research/gen_ai/trial.rb | 14 ++- .../evidence/gen_ai/evaluation/creator.rb | 42 +++++++ .../gen_ai/evaluation/data_fetcher.rb | 2 +- .../gen_ai/evaluation/feedback_evaluator.rb | 2 + .../gen_ai/calculate_results_worker.rb | 6 +- .../gen_ai/run_feedback_evaluation_worker.rb | 17 +++ .../gen_ai/run_trial_evaluation_worker.rb | 35 ++++++ ...te_research_gen_ai_feedback_evaluations.rb | 16 +++ 12 files changed, 290 insertions(+), 6 deletions(-) create mode 100644 services/QuillLMS/db/migrate/20250130191426_create_research_gen_ai_feedback_evaluations.evidence.rb create mode 100644 services/QuillLMS/engines/evidence/app/models/evidence/research/gen_ai/feedback_evaluation.rb create mode 100644 services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/creator.rb create mode 100644 services/QuillLMS/engines/evidence/app/workers/evidence/research/gen_ai/run_feedback_evaluation_worker.rb create mode 100644 services/QuillLMS/engines/evidence/app/workers/evidence/research/gen_ai/run_trial_evaluation_worker.rb create mode 100644 services/QuillLMS/engines/evidence/db/migrate/20250123202558_create_research_gen_ai_feedback_evaluations.rb diff --git a/services/QuillLMS/db/migrate/20250130191426_create_research_gen_ai_feedback_evaluations.evidence.rb b/services/QuillLMS/db/migrate/20250130191426_create_research_gen_ai_feedback_evaluations.evidence.rb new file mode 100644 index 00000000000..c814f36e757 --- /dev/null +++ b/services/QuillLMS/db/migrate/20250130191426_create_research_gen_ai_feedback_evaluations.evidence.rb @@ -0,0 +1,17 @@ +# This migration comes from evidence (originally 20250123202558) +class CreateResearchGenAIFeedbackEvaluations < ActiveRecord::Migration[7.1] + def change + create_table :evidence_research_gen_ai_feedback_evaluations do |t| + t.integer :trial_id, null: false + t.integer :version, null: false + t.integer :llm_example_id, null: false + t.boolean :optimal, null: false + t.string :errored_checks, array: true, default: [] + + t.timestamps + end + + add_index :evidence_research_gen_ai_feedback_evaluations, :trial_id + add_index :evidence_research_gen_ai_feedback_evaluations, :llm_example_id + end +end diff --git a/services/QuillLMS/db/structure.sql b/services/QuillLMS/db/structure.sql index afede47989d..22bee6a2601 100644 --- a/services/QuillLMS/db/structure.sql +++ b/services/QuillLMS/db/structure.sql @@ -2791,6 +2791,41 @@ CREATE SEQUENCE public.evidence_automl_models_id_seq ALTER SEQUENCE public.evidence_automl_models_id_seq OWNED BY public.evidence_automl_models.id; +-- +-- Name: evidence_gen_ai_feedback_evaluation; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.evidence_gen_ai_feedback_evaluation ( + id bigint NOT NULL, + trial_id integer NOT NULL, + version integer NOT NULL, + llm_example_id integer NOT NULL, + optimal boolean NOT NULL, + rules character varying[] DEFAULT '{}'::character varying[], + created_at timestamp(6) without time zone NOT NULL, + updated_at timestamp(6) without time zone NOT NULL +); + + +-- +-- Name: evidence_gen_ai_feedback_evaluation_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.evidence_gen_ai_feedback_evaluation_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: evidence_gen_ai_feedback_evaluation_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.evidence_gen_ai_feedback_evaluation_id_seq OWNED BY public.evidence_gen_ai_feedback_evaluation.id; + + -- -- Name: evidence_gen_ai_highlight_groups; Type: TABLE; Schema: public; Owner: - -- @@ -3244,6 +3279,41 @@ CREATE SEQUENCE public.evidence_research_gen_ai_datasets_id_seq ALTER SEQUENCE public.evidence_research_gen_ai_datasets_id_seq OWNED BY public.evidence_research_gen_ai_datasets.id; +-- +-- Name: evidence_research_gen_ai_feedback_evaluations; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.evidence_research_gen_ai_feedback_evaluations ( + id bigint NOT NULL, + trial_id integer NOT NULL, + version integer NOT NULL, + llm_example_id integer NOT NULL, + optimal boolean NOT NULL, + errored_checks character varying[] DEFAULT '{}'::character varying[], + created_at timestamp(6) without time zone NOT NULL, + updated_at timestamp(6) without time zone NOT NULL +); + + +-- +-- Name: evidence_research_gen_ai_feedback_evaluations_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.evidence_research_gen_ai_feedback_evaluations_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: evidence_research_gen_ai_feedback_evaluations_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.evidence_research_gen_ai_feedback_evaluations_id_seq OWNED BY public.evidence_research_gen_ai_feedback_evaluations.id; + + -- -- Name: evidence_research_gen_ai_g_eval_scores; Type: TABLE; Schema: public; Owner: - -- @@ -7254,6 +7324,13 @@ ALTER TABLE ONLY public.evidence_research_gen_ai_dataset_relevant_texts ALTER CO ALTER TABLE ONLY public.evidence_research_gen_ai_datasets ALTER COLUMN id SET DEFAULT nextval('public.evidence_research_gen_ai_datasets_id_seq'::regclass); +-- +-- Name: evidence_research_gen_ai_feedback_evaluations id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.evidence_research_gen_ai_feedback_evaluations ALTER COLUMN id SET DEFAULT nextval('public.evidence_research_gen_ai_feedback_evaluations_id_seq'::regclass); + + -- -- Name: evidence_research_gen_ai_g_eval_scores id; Type: DEFAULT; Schema: public; Owner: - -- @@ -8620,6 +8697,14 @@ ALTER TABLE ONLY public.evidence_research_gen_ai_datasets ADD CONSTRAINT evidence_research_gen_ai_datasets_pkey PRIMARY KEY (id); +-- +-- Name: evidence_research_gen_ai_feedback_evaluations evidence_research_gen_ai_feedback_evaluations_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.evidence_research_gen_ai_feedback_evaluations + ADD CONSTRAINT evidence_research_gen_ai_feedback_evaluations_pkey PRIMARY KEY (id); + + -- -- Name: evidence_research_gen_ai_g_eval_scores evidence_research_gen_ai_g_eval_scores_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- @@ -9469,6 +9554,13 @@ CREATE UNIQUE INDEX feedback_history_ratings_uniqueness ON public.feedback_histo CREATE INDEX idx_on_classroom_unit_id_8502333889 ON public.student_learning_sequence_activities USING btree (classroom_unit_id); +-- +-- Name: idx_on_llm_example_id_774518eafd; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_on_llm_example_id_774518eafd ON public.evidence_research_gen_ai_feedback_evaluations USING btree (llm_example_id); + + -- -- Name: idx_on_student_learning_sequence_id_63827699e9; Type: INDEX; Schema: public; Owner: - -- @@ -9483,6 +9575,13 @@ CREATE INDEX idx_on_student_learning_sequence_id_63827699e9 ON public.student_le CREATE UNIQUE INDEX idx_on_student_learning_sequence_id_classroom_unit__84e420e79d ON public.student_learning_sequence_activities USING btree (student_learning_sequence_id, classroom_unit_id, activity_id); +-- +-- Name: idx_on_trial_id_eee55ce93c; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_on_trial_id_eee55ce93c ON public.evidence_research_gen_ai_feedback_evaluations USING btree (trial_id); + + -- -- Name: idx_on_user_id_initial_classroom_unit_id_initial_ac_17357537ed; Type: INDEX; Schema: public; Owner: - -- @@ -11981,9 +12080,12 @@ ALTER TABLE ONLY public.learn_worlds_account_course_events -- PostgreSQL database dump complete -- -SET search_path TO "$user", public; +SET search_path TO "$user", public, heroku_ext; INSERT INTO "schema_migrations" (version) VALUES +('20250130191426'), +('20250109202521'), +('20250109151117'), ('20241211155124'), ('20241211152410'), ('20241211151409'), diff --git a/services/QuillLMS/engines/evidence/README.rdoc b/services/QuillLMS/engines/evidence/README.rdoc index 107def095f7..30197ee5a26 100644 --- a/services/QuillLMS/engines/evidence/README.rdoc +++ b/services/QuillLMS/engines/evidence/README.rdoc @@ -47,10 +47,10 @@ After generating the scaffold You run migrations in the parent app, not in the engine. So if you add a migration, run this in the parent app: -`rake evidence:install:migrations` +`bin/rails evidence:install:migrations` If you are hitting an issue in which running `rspec` keeps telling you have a pending migration (and you've run the LMS migrations), run the following in the evidence directory: -`rake db:schema:load RAILS_ENV=test` +`bin/rails db:schema:load RAILS_ENV=test` See the Rails guide for more details. diff --git a/services/QuillLMS/engines/evidence/app/models/evidence/research/gen_ai/feedback_evaluation.rb b/services/QuillLMS/engines/evidence/app/models/evidence/research/gen_ai/feedback_evaluation.rb new file mode 100644 index 00000000000..ab7fb56551e --- /dev/null +++ b/services/QuillLMS/engines/evidence/app/models/evidence/research/gen_ai/feedback_evaluation.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +# == Schema Information +# +# Table name: evidence_research_gen_ai_feedback_evaluations +# +# id :bigint not null, primary key +# errored_checks :string default([]), is an Array +# optimal :boolean not null +# version :integer not null +# created_at :datetime not null +# updated_at :datetime not null +# llm_example_id :integer not null +# trial_id :integer not null +# +# Indexes +# +# idx_on_llm_example_id_774518eafd (llm_example_id) +# idx_on_trial_id_eee55ce93c (trial_id) +# +module Evidence + module Research + module GenAI + class FeedbackEvaluation < ApplicationRecord + belongs_to :trial + belongs_to :llm_example + + attr_readonly :trial_id, :version, :llm_example_id, :optimal, :errors + + validates :trial_id, :llm_example_id, presence: true + validates :optimal, inclusion: { in: [true,false] } + + scope :suboptimal, -> { where(optimal: false) } + end + end + end +end diff --git a/services/QuillLMS/engines/evidence/app/models/evidence/research/gen_ai/trial.rb b/services/QuillLMS/engines/evidence/app/models/evidence/research/gen_ai/trial.rb index 5ebe27c61c6..d628c2616ae 100644 --- a/services/QuillLMS/engines/evidence/app/models/evidence/research/gen_ai/trial.rb +++ b/services/QuillLMS/engines/evidence/app/models/evidence/research/gen_ai/trial.rb @@ -41,6 +41,7 @@ class Trial < ApplicationRecord has_many :llm_examples, -> { includes(:test_example).order(:id) } has_many :test_examples, -> { order(:id) }, through: :dataset + has_many :feedback_evaluations validates :llm_id, :llm_prompt_id, :dataset_id, presence: true validates :status, presence: true, inclusion: { in: STATUSES } @@ -70,7 +71,9 @@ class Trial < ApplicationRecord :suboptimal_correct, :optimal_accuracy, :suboptimal_accuracy, - :weighted_accuracy + :weighted_accuracy, + :feedback_failure_count, + :feedback_failure_percentage attr_readonly :llm_id, :llm_prompt_id, :dataset_id, :temperature @@ -119,6 +122,15 @@ def set_trial_start_time = update!(trial_start_time: Time.zone.now) def set_trial_duration = update!(trial_duration: Time.zone.now - Time.zone.parse(trial_start_time)) def set_status = trial_errors.empty? ? completed! : failed! + def set_feedback_failure_metrics + feedback_failure_count = feedback_evaluations.suboptimal.count + total = feedback_evaluations.count + + feedback_failure_percentage = (feedback_failure_count / total.to_f).round(4) + + update(feedback_failure_count:, feedback_failure_percentage:) + end + private def set_accuracy optimal_correct = compute_optimal_correct suboptimal_correct = compute_suboptimal_correct diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/creator.rb b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/creator.rb new file mode 100644 index 00000000000..b5410dd8cc1 --- /dev/null +++ b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/creator.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +module Evidence + module GenAI + module Evaluation + class Creator < ApplicationService + + attr_reader :trial_id, :version, :llm_example_id + + def initialize(trial_id:, version:, llm_example_id:) + @trial_id = trial_id + @version = version + @llm_example_id = llm_example_id + end + + def run + Evidence::Research::GenAI::FeedbackEvaluation.create( + trial_id:, + version:, + llm_example_id:, + optimal:, + errored_checks: + ) + end + + private def optimal = evaluation.optimal + private def errored_checks = evaluation.errors + + private def evaluation + @evaluation ||= FeedbackEvaluator.run(feedback:, entry:) + end + + private def feedback = llm_example.llm_feedback + private def entry = llm_example.student_response + + private def llm_example + @llm_example ||= Evidence::Research::GenAI::LLMExample.find(llm_example_id) + end + end + end + end +end diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/data_fetcher.rb b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/data_fetcher.rb index de93beab2b2..5f8bc1a1ee0 100644 --- a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/data_fetcher.rb +++ b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/data_fetcher.rb @@ -11,7 +11,7 @@ class DataFetcher < ApplicationService SOLDIERS_BECAUSE = 'underage_soldiers_because.csv' HAITIAN_BECAUSE = 'new_haitian_because.csv' - DEFAULT_FILE = WHALES_SO + DEFAULT_FILE = SOLDIERS_BECAUSE CSV_FILE_PATH = "#{Evidence::Engine.root}/app/services/evidence/gen_ai/evaluation/data/%s" attr_reader :rule, :optimal diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/feedback_evaluator.rb b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/feedback_evaluator.rb index 9a93dabe795..c1b1018c432 100644 --- a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/feedback_evaluator.rb +++ b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/feedback_evaluator.rb @@ -6,6 +6,8 @@ module Evaluation class FeedbackEvaluator < ApplicationService Result = Data.define(:optimal, :errors) + VERSION = 1 + CHECKS = [ Tone::Checker, Grammar::Checker, diff --git a/services/QuillLMS/engines/evidence/app/workers/evidence/research/gen_ai/calculate_results_worker.rb b/services/QuillLMS/engines/evidence/app/workers/evidence/research/gen_ai/calculate_results_worker.rb index 12ccd515f4a..d042ee0092c 100644 --- a/services/QuillLMS/engines/evidence/app/workers/evidence/research/gen_ai/calculate_results_worker.rb +++ b/services/QuillLMS/engines/evidence/app/workers/evidence/research/gen_ai/calculate_results_worker.rb @@ -14,7 +14,11 @@ def perform(trial_id) trial = Trial.find(trial_id) trial.set_results trial.set_evaluation_start_time - GEvalScoresFetcher.run(trial) if trial.generative? + + if trial.generative? + GEvalScoresFetcher.run(trial) + RunTrialEvaluationWorker.perform_async(trial_id) + end end end end diff --git a/services/QuillLMS/engines/evidence/app/workers/evidence/research/gen_ai/run_feedback_evaluation_worker.rb b/services/QuillLMS/engines/evidence/app/workers/evidence/research/gen_ai/run_feedback_evaluation_worker.rb new file mode 100644 index 00000000000..b6a992c7a75 --- /dev/null +++ b/services/QuillLMS/engines/evidence/app/workers/evidence/research/gen_ai/run_feedback_evaluation_worker.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Evidence + module Research + module GenAI + class RunFeedbackEvaluationWorker + include Evidence.sidekiq_module + + sidekiq_options retry: 3, queue: 'default' + + def perform(trial_id, version, llm_example_id) + Evidence::GenAI::Evaluation::Creator.run(trial_id:, version:, llm_example_id:) + end + end + end + end +end diff --git a/services/QuillLMS/engines/evidence/app/workers/evidence/research/gen_ai/run_trial_evaluation_worker.rb b/services/QuillLMS/engines/evidence/app/workers/evidence/research/gen_ai/run_trial_evaluation_worker.rb new file mode 100644 index 00000000000..0b386692d12 --- /dev/null +++ b/services/QuillLMS/engines/evidence/app/workers/evidence/research/gen_ai/run_trial_evaluation_worker.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Evidence + module Research + module GenAI + class RunTrialEvaluationWorker + include Evidence.sidekiq_module + + sidekiq_options retry: 3, queue: 'default' + + VERSION = Evidence::GenAI::Evaluation::FeedbackEvaluator::VERSION + + def perform(trial_id) + batch = Sidekiq::Batch.new + batch.on(:complete, self.class, trial_id: trial_id) + + trial = Trial.find(trial_id) + + batch.jobs do + # Note, we only run the Evaluator on suboptimal feedback + trial.llm_examples.suboptimal.each do |llm_example| + RunFeedbackEvaluationWorker.perform_async(trial_id, VERSION, llm_example.id) + end + end + end + + def on_complete(_status, options) + Trial + .find(options['trial_id']) + .set_feedback_failure_metrics + end + end + end + end +end diff --git a/services/QuillLMS/engines/evidence/db/migrate/20250123202558_create_research_gen_ai_feedback_evaluations.rb b/services/QuillLMS/engines/evidence/db/migrate/20250123202558_create_research_gen_ai_feedback_evaluations.rb new file mode 100644 index 00000000000..5961f109943 --- /dev/null +++ b/services/QuillLMS/engines/evidence/db/migrate/20250123202558_create_research_gen_ai_feedback_evaluations.rb @@ -0,0 +1,16 @@ +class CreateResearchGenAIFeedbackEvaluations < ActiveRecord::Migration[7.1] + def change + create_table :evidence_research_gen_ai_feedback_evaluations do |t| + t.integer :trial_id, null: false + t.integer :version, null: false + t.integer :llm_example_id, null: false + t.boolean :optimal, null: false + t.string :errored_checks, array: true, default: [] + + t.timestamps + end + + add_index :evidence_research_gen_ai_feedback_evaluations, :trial_id + add_index :evidence_research_gen_ai_feedback_evaluations, :llm_example_id + end +end From 79a72e3cdb3717531859534bdfe544ed7858c37e Mon Sep 17 00:00:00 2001 From: Daniel Drabik Date: Thu, 30 Jan 2025 16:16:05 -0500 Subject: [PATCH 30/51] Update controller endpoints for frontend use. --- .../QuillLMS/engines/evidence/README.rdoc | 2 +- .../gen_ai/llm_examples_controller.rb | 2 +- .../research/gen_ai/feedback_evaluation.rb | 7 ++ .../evidence/research/gen_ai/llm_example.rb | 5 +- .../evidence/spec/dummy/db/structure.sql | 72 +++++++++++++++++++ 5 files changed, 85 insertions(+), 3 deletions(-) diff --git a/services/QuillLMS/engines/evidence/README.rdoc b/services/QuillLMS/engines/evidence/README.rdoc index 30197ee5a26..fe5a934b082 100644 --- a/services/QuillLMS/engines/evidence/README.rdoc +++ b/services/QuillLMS/engines/evidence/README.rdoc @@ -51,6 +51,6 @@ You run migrations in the parent app, not in the engine. So if you add a migrati If you are hitting an issue in which running `rspec` keeps telling you have a pending migration (and you've run the LMS migrations), run the following in the evidence directory: -`bin/rails db:schema:load RAILS_ENV=test` +`bundle exec rake db:schema:load RAILS_ENV=test` See the Rails guide for more details. diff --git a/services/QuillLMS/engines/evidence/app/controllers/evidence/research/gen_ai/llm_examples_controller.rb b/services/QuillLMS/engines/evidence/app/controllers/evidence/research/gen_ai/llm_examples_controller.rb index 4eaaaa6698d..69d8b7fc46f 100644 --- a/services/QuillLMS/engines/evidence/app/controllers/evidence/research/gen_ai/llm_examples_controller.rb +++ b/services/QuillLMS/engines/evidence/app/controllers/evidence/research/gen_ai/llm_examples_controller.rb @@ -5,7 +5,7 @@ module Research module GenAI class LLMExamplesController < ApplicationController def index - render json: { llm_examples: trial.llm_examples.order(:id) } + render json: { llm_examples: trial.llm_examples.includes(:feedback_evaluation).order(:id) } end private def trial = @trial ||= Trial.find(params[:trial_id]) diff --git a/services/QuillLMS/engines/evidence/app/models/evidence/research/gen_ai/feedback_evaluation.rb b/services/QuillLMS/engines/evidence/app/models/evidence/research/gen_ai/feedback_evaluation.rb index ab7fb56551e..ad8453f008d 100644 --- a/services/QuillLMS/engines/evidence/app/models/evidence/research/gen_ai/feedback_evaluation.rb +++ b/services/QuillLMS/engines/evidence/app/models/evidence/research/gen_ai/feedback_evaluation.rb @@ -31,6 +31,13 @@ class FeedbackEvaluation < ApplicationRecord validates :optimal, inclusion: { in: [true,false] } scope :suboptimal, -> { where(optimal: false) } + + def serializable_hash(options = nil) + options ||= {} + super(options.reverse_merge( + only: [:optimal, :errored_checks, :version] + )) + end end end end diff --git a/services/QuillLMS/engines/evidence/app/models/evidence/research/gen_ai/llm_example.rb b/services/QuillLMS/engines/evidence/app/models/evidence/research/gen_ai/llm_example.rb index cc56fc648e2..139cc233356 100644 --- a/services/QuillLMS/engines/evidence/app/models/evidence/research/gen_ai/llm_example.rb +++ b/services/QuillLMS/engines/evidence/app/models/evidence/research/gen_ai/llm_example.rb @@ -23,6 +23,9 @@ class LLMExample < ApplicationRecord belongs_to :trial belongs_to :test_example + has_many :feedback_evaluations + has_one :feedback_evaluation, -> { order(created_at: :desc) }, class_name: "FeedbackEvaluation" + validates :raw_text, presence: true validates :llm_feedback, presence: true validates :test_example_id, presence: true @@ -36,7 +39,7 @@ class LLMExample < ApplicationRecord def serializable_hash(options = nil) options ||= {} super(options.reverse_merge( - include: [:test_example], + include: [:test_example, :feedback_evaluation], methods: [:optimal_or_suboptimal_match?, :optimal?, :suboptimal?, :rag_label] )) end diff --git a/services/QuillLMS/engines/evidence/spec/dummy/db/structure.sql b/services/QuillLMS/engines/evidence/spec/dummy/db/structure.sql index 85912e81883..da6442ea623 100644 --- a/services/QuillLMS/engines/evidence/spec/dummy/db/structure.sql +++ b/services/QuillLMS/engines/evidence/spec/dummy/db/structure.sql @@ -9,6 +9,13 @@ SET xmloption = content; SET client_min_messages = warning; SET row_security = off; +-- +-- Name: public; Type: SCHEMA; Schema: -; Owner: - +-- + +-- *not* creating schema, since initdb creates it + + -- -- Name: vector; Type: EXTENSION; Schema: -; Owner: - -- @@ -1171,6 +1178,41 @@ CREATE SEQUENCE public.evidence_research_gen_ai_datasets_id_seq ALTER SEQUENCE public.evidence_research_gen_ai_datasets_id_seq OWNED BY public.evidence_research_gen_ai_datasets.id; +-- +-- Name: evidence_research_gen_ai_feedback_evaluations; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.evidence_research_gen_ai_feedback_evaluations ( + id bigint NOT NULL, + trial_id integer NOT NULL, + version integer NOT NULL, + llm_example_id integer NOT NULL, + optimal boolean NOT NULL, + errored_checks character varying[] DEFAULT '{}'::character varying[], + created_at timestamp(6) without time zone NOT NULL, + updated_at timestamp(6) without time zone NOT NULL +); + + +-- +-- Name: evidence_research_gen_ai_feedback_evaluations_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.evidence_research_gen_ai_feedback_evaluations_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: evidence_research_gen_ai_feedback_evaluations_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.evidence_research_gen_ai_feedback_evaluations_id_seq OWNED BY public.evidence_research_gen_ai_feedback_evaluations.id; + + -- -- Name: evidence_research_gen_ai_g_eval_scores; Type: TABLE; Schema: public; Owner: - -- @@ -2093,6 +2135,13 @@ ALTER TABLE ONLY public.evidence_research_gen_ai_dataset_relevant_texts ALTER CO ALTER TABLE ONLY public.evidence_research_gen_ai_datasets ALTER COLUMN id SET DEFAULT nextval('public.evidence_research_gen_ai_datasets_id_seq'::regclass); +-- +-- Name: evidence_research_gen_ai_feedback_evaluations id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.evidence_research_gen_ai_feedback_evaluations ALTER COLUMN id SET DEFAULT nextval('public.evidence_research_gen_ai_feedback_evaluations_id_seq'::regclass); + + -- -- Name: evidence_research_gen_ai_g_eval_scores id; Type: DEFAULT; Schema: public; Owner: - -- @@ -2497,6 +2546,14 @@ ALTER TABLE ONLY public.evidence_research_gen_ai_datasets ADD CONSTRAINT evidence_research_gen_ai_datasets_pkey PRIMARY KEY (id); +-- +-- Name: evidence_research_gen_ai_feedback_evaluations evidence_research_gen_ai_feedback_evaluations_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.evidence_research_gen_ai_feedback_evaluations + ADD CONSTRAINT evidence_research_gen_ai_feedback_evaluations_pkey PRIMARY KEY (id); + + -- -- Name: evidence_research_gen_ai_g_eval_scores evidence_research_gen_ai_g_eval_scores_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- @@ -2679,6 +2736,20 @@ CREATE UNIQUE INDEX comprehension_turking_sessions_activity_uid ON public.compre CREATE INDEX comprehension_turking_sessions_turking_id ON public.comprehension_turking_round_activity_sessions USING btree (turking_round_id); +-- +-- Name: idx_on_llm_example_id_774518eafd; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_on_llm_example_id_774518eafd ON public.evidence_research_gen_ai_feedback_evaluations USING btree (llm_example_id); + + +-- +-- Name: idx_on_trial_id_eee55ce93c; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_on_trial_id_eee55ce93c ON public.evidence_research_gen_ai_feedback_evaluations USING btree (trial_id); + + -- -- Name: index_change_logs_on_changed_record_id; Type: INDEX; Schema: public; Owner: - -- @@ -2889,6 +2960,7 @@ ALTER TABLE ONLY public.comprehension_regex_rules SET search_path TO "$user", public; INSERT INTO "schema_migrations" (version) VALUES +('20250123202558'), ('20241211154437'), ('20241211151946'), ('20241211151245'), From e209a025643812c2111ac37e117fb24129c2d76e Mon Sep 17 00:00:00 2001 From: Daniel Drabik Date: Fri, 31 Jan 2025 09:48:15 -0500 Subject: [PATCH 31/51] Whitespace to trigger build. --- .../evidence/app/services/evidence/gen_ai/evaluation/creator.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/creator.rb b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/creator.rb index b5410dd8cc1..8aac8e64e30 100644 --- a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/creator.rb +++ b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/creator.rb @@ -13,6 +13,7 @@ def initialize(trial_id:, version:, llm_example_id:) @llm_example_id = llm_example_id end + def run Evidence::Research::GenAI::FeedbackEvaluation.create( trial_id:, From 9ec3bb70af79d3dfc660add4f3ee59e05321cc91 Mon Sep 17 00:00:00 2001 From: Daniel Drabik Date: Fri, 31 Jan 2025 13:29:24 -0500 Subject: [PATCH 32/51] Add creator spec. --- .../evidence/gen_ai/evaluation/creator.rb | 2 +- .../gen_ai/evaluation/creator_spec.rb | 51 +++++++++++++++++++ 2 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 services/QuillLMS/engines/evidence/spec/services/evidence/gen_ai/evaluation/creator_spec.rb diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/creator.rb b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/creator.rb index 8aac8e64e30..d56ef93483d 100644 --- a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/creator.rb +++ b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/creator.rb @@ -15,7 +15,7 @@ def initialize(trial_id:, version:, llm_example_id:) def run - Evidence::Research::GenAI::FeedbackEvaluation.create( + Evidence::Research::GenAI::FeedbackEvaluation.create!( trial_id:, version:, llm_example_id:, diff --git a/services/QuillLMS/engines/evidence/spec/services/evidence/gen_ai/evaluation/creator_spec.rb b/services/QuillLMS/engines/evidence/spec/services/evidence/gen_ai/evaluation/creator_spec.rb new file mode 100644 index 00000000000..7cce20f4ea0 --- /dev/null +++ b/services/QuillLMS/engines/evidence/spec/services/evidence/gen_ai/evaluation/creator_spec.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +require 'rails_helper' + +RSpec.describe Evidence::GenAI::Evaluation::Creator do + let(:trial_id) { 1 } + let(:version) { 1 } + let(:llm_example_id) { 100 } + let(:llm_example) { double(Evidence::Research::GenAI::LLMExample, llm_feedback: 'Good job', student_response: 'Answer') } + let(:evaluation) { double(Evidence::GenAI::Evaluation::FeedbackEvaluator, optimal: true, errors: []) } + + before do + allow(Evidence::Research::GenAI::LLMExample).to receive(:find).with(llm_example_id).and_return(llm_example) + allow(Evidence::GenAI::Evaluation::FeedbackEvaluator).to receive(:run).with(feedback: llm_example.llm_feedback, entry: llm_example.student_response).and_return(evaluation) + allow(Evidence::Research::GenAI::FeedbackEvaluation).to receive(:create) + end + + describe '#run' do + subject { described_class.run(trial_id:, version:, llm_example_id:) } + + it 'fetches the LLM example' do + expect(Evidence::Research::GenAI::LLMExample).to receive(:find).with(llm_example_id) + subject + end + + it 'evaluates the feedback' do + expect(Evidence::GenAI::Evaluation::FeedbackEvaluator).to receive(:run).with(feedback: llm_example.llm_feedback, entry: llm_example.student_response) + subject + end + + it 'creates a FeedbackEvaluation record' do + expect(Evidence::Research::GenAI::FeedbackEvaluation).to receive(:create!).with( + trial_id:, + version: , + llm_example_id:, + optimal: true, + errored_checks: [] + ) + subject + end + + describe 'create feedback_evaluation' do + let(:llm_example) { create(:evidence_research_gen_ai_llm_example) } + let(:llm_example_id) { llm_example.id } + let(:trial) { create(:evidence_research_gen_ai_trial) } + let(:trial_id) { trial.id } + + it { expect(subject.valid?).to be true } + end + end +end From 2c0dc024318455033d9ed468cb03ec49e3cab731 Mon Sep 17 00:00:00 2001 From: Daniel Drabik Date: Fri, 31 Jan 2025 14:27:00 -0500 Subject: [PATCH 33/51] Add backstop tests. --- ...ch_gen_ai_feedback_evaluations.evidence.rb | 2 + .../research/gen_ai/feedback_evaluation.rb | 2 +- .../evidence/research/gen_ai/llm_example.rb | 2 +- .../models/evidence/research/gen_ai/trial.rb | 2 +- .../evidence/gen_ai/evaluation/creator.rb | 2 - .../gen_ai/calculate_results_worker.rb | 8 ++-- .../gen_ai/run_trial_evaluation_worker.rb | 4 +- ...te_research_gen_ai_feedback_evaluations.rb | 2 + .../research/gen_ai/feedback_evaluations.rb | 22 +++++++++++ .../evidence/research/gen_ai/trial_spec.rb | 14 +++++++ .../gen_ai/evaluation/creator_spec.rb | 2 +- .../gen_ai/calculate_results_worker_spec.rb | 2 + .../run_feedback_evaluation_worker_spec.rb | 24 ++++++++++++ .../run_trial_evaluation_worker_spec.rb | 37 +++++++++++++++++++ 14 files changed, 113 insertions(+), 12 deletions(-) create mode 100644 services/QuillLMS/engines/evidence/spec/factories/evidence/research/gen_ai/feedback_evaluations.rb create mode 100644 services/QuillLMS/engines/evidence/spec/workers/evidence/research/gen_ai/run_feedback_evaluation_worker_spec.rb create mode 100644 services/QuillLMS/engines/evidence/spec/workers/evidence/research/gen_ai/run_trial_evaluation_worker_spec.rb diff --git a/services/QuillLMS/db/migrate/20250130191426_create_research_gen_ai_feedback_evaluations.evidence.rb b/services/QuillLMS/db/migrate/20250130191426_create_research_gen_ai_feedback_evaluations.evidence.rb index c814f36e757..2be80a48b54 100644 --- a/services/QuillLMS/db/migrate/20250130191426_create_research_gen_ai_feedback_evaluations.evidence.rb +++ b/services/QuillLMS/db/migrate/20250130191426_create_research_gen_ai_feedback_evaluations.evidence.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # This migration comes from evidence (originally 20250123202558) class CreateResearchGenAIFeedbackEvaluations < ActiveRecord::Migration[7.1] def change diff --git a/services/QuillLMS/engines/evidence/app/models/evidence/research/gen_ai/feedback_evaluation.rb b/services/QuillLMS/engines/evidence/app/models/evidence/research/gen_ai/feedback_evaluation.rb index ad8453f008d..058a4c902a5 100644 --- a/services/QuillLMS/engines/evidence/app/models/evidence/research/gen_ai/feedback_evaluation.rb +++ b/services/QuillLMS/engines/evidence/app/models/evidence/research/gen_ai/feedback_evaluation.rb @@ -28,7 +28,7 @@ class FeedbackEvaluation < ApplicationRecord attr_readonly :trial_id, :version, :llm_example_id, :optimal, :errors validates :trial_id, :llm_example_id, presence: true - validates :optimal, inclusion: { in: [true,false] } + validates :optimal, inclusion: { in: [true, false] } scope :suboptimal, -> { where(optimal: false) } diff --git a/services/QuillLMS/engines/evidence/app/models/evidence/research/gen_ai/llm_example.rb b/services/QuillLMS/engines/evidence/app/models/evidence/research/gen_ai/llm_example.rb index 139cc233356..5f4815c4406 100644 --- a/services/QuillLMS/engines/evidence/app/models/evidence/research/gen_ai/llm_example.rb +++ b/services/QuillLMS/engines/evidence/app/models/evidence/research/gen_ai/llm_example.rb @@ -24,7 +24,7 @@ class LLMExample < ApplicationRecord belongs_to :test_example has_many :feedback_evaluations - has_one :feedback_evaluation, -> { order(created_at: :desc) }, class_name: "FeedbackEvaluation" + has_one :feedback_evaluation, -> { order(created_at: :desc) }, class_name: 'FeedbackEvaluation' validates :raw_text, presence: true validates :llm_feedback, presence: true diff --git a/services/QuillLMS/engines/evidence/app/models/evidence/research/gen_ai/trial.rb b/services/QuillLMS/engines/evidence/app/models/evidence/research/gen_ai/trial.rb index d628c2616ae..63c9fb48935 100644 --- a/services/QuillLMS/engines/evidence/app/models/evidence/research/gen_ai/trial.rb +++ b/services/QuillLMS/engines/evidence/app/models/evidence/research/gen_ai/trial.rb @@ -122,7 +122,7 @@ def set_trial_start_time = update!(trial_start_time: Time.zone.now) def set_trial_duration = update!(trial_duration: Time.zone.now - Time.zone.parse(trial_start_time)) def set_status = trial_errors.empty? ? completed! : failed! - def set_feedback_failure_metrics + def set_feedback_evaluation_metrics feedback_failure_count = feedback_evaluations.suboptimal.count total = feedback_evaluations.count diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/creator.rb b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/creator.rb index d56ef93483d..496e4e1684a 100644 --- a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/creator.rb +++ b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/creator.rb @@ -4,7 +4,6 @@ module Evidence module GenAI module Evaluation class Creator < ApplicationService - attr_reader :trial_id, :version, :llm_example_id def initialize(trial_id:, version:, llm_example_id:) @@ -13,7 +12,6 @@ def initialize(trial_id:, version:, llm_example_id:) @llm_example_id = llm_example_id end - def run Evidence::Research::GenAI::FeedbackEvaluation.create!( trial_id:, diff --git a/services/QuillLMS/engines/evidence/app/workers/evidence/research/gen_ai/calculate_results_worker.rb b/services/QuillLMS/engines/evidence/app/workers/evidence/research/gen_ai/calculate_results_worker.rb index d042ee0092c..8cab1d6a380 100644 --- a/services/QuillLMS/engines/evidence/app/workers/evidence/research/gen_ai/calculate_results_worker.rb +++ b/services/QuillLMS/engines/evidence/app/workers/evidence/research/gen_ai/calculate_results_worker.rb @@ -15,10 +15,10 @@ def perform(trial_id) trial.set_results trial.set_evaluation_start_time - if trial.generative? - GEvalScoresFetcher.run(trial) - RunTrialEvaluationWorker.perform_async(trial_id) - end + return unless trial.generative? + + GEvalScoresFetcher.run(trial) + RunTrialEvaluationWorker.perform_async(trial_id) end end end diff --git a/services/QuillLMS/engines/evidence/app/workers/evidence/research/gen_ai/run_trial_evaluation_worker.rb b/services/QuillLMS/engines/evidence/app/workers/evidence/research/gen_ai/run_trial_evaluation_worker.rb index 0b386692d12..7eebb2fa497 100644 --- a/services/QuillLMS/engines/evidence/app/workers/evidence/research/gen_ai/run_trial_evaluation_worker.rb +++ b/services/QuillLMS/engines/evidence/app/workers/evidence/research/gen_ai/run_trial_evaluation_worker.rb @@ -12,7 +12,7 @@ class RunTrialEvaluationWorker def perform(trial_id) batch = Sidekiq::Batch.new - batch.on(:complete, self.class, trial_id: trial_id) + batch.on(:complete, self.class, trial_id:) trial = Trial.find(trial_id) @@ -27,7 +27,7 @@ def perform(trial_id) def on_complete(_status, options) Trial .find(options['trial_id']) - .set_feedback_failure_metrics + .set_feedback_evaluation_metrics end end end diff --git a/services/QuillLMS/engines/evidence/db/migrate/20250123202558_create_research_gen_ai_feedback_evaluations.rb b/services/QuillLMS/engines/evidence/db/migrate/20250123202558_create_research_gen_ai_feedback_evaluations.rb index 5961f109943..28ef4611716 100644 --- a/services/QuillLMS/engines/evidence/db/migrate/20250123202558_create_research_gen_ai_feedback_evaluations.rb +++ b/services/QuillLMS/engines/evidence/db/migrate/20250123202558_create_research_gen_ai_feedback_evaluations.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + class CreateResearchGenAIFeedbackEvaluations < ActiveRecord::Migration[7.1] def change create_table :evidence_research_gen_ai_feedback_evaluations do |t| diff --git a/services/QuillLMS/engines/evidence/spec/factories/evidence/research/gen_ai/feedback_evaluations.rb b/services/QuillLMS/engines/evidence/spec/factories/evidence/research/gen_ai/feedback_evaluations.rb new file mode 100644 index 00000000000..b836abf19b0 --- /dev/null +++ b/services/QuillLMS/engines/evidence/spec/factories/evidence/research/gen_ai/feedback_evaluations.rb @@ -0,0 +1,22 @@ + +module Evidence + module Research + module GenAI + FactoryBot.define do + factory :evidence_research_gen_ai_feedback_evaluation, class: 'Evidence::Research::GenAI::FeedbackEvaluation' do + version { 1 } + llm_example { association :evidence_research_gen_ai_llm_example } + trial { association :evidence_research_gen_ai_trial } + optimal true + errored_checks [] + + trait(:optimal) { optimal true } + trait(:suboptimal) do + optimal false + errored_checks ['tone'] + end + end + end + end + end +end diff --git a/services/QuillLMS/engines/evidence/spec/models/evidence/research/gen_ai/trial_spec.rb b/services/QuillLMS/engines/evidence/spec/models/evidence/research/gen_ai/trial_spec.rb index e2e6c8ebfce..c2ccb81d351 100644 --- a/services/QuillLMS/engines/evidence/spec/models/evidence/research/gen_ai/trial_spec.rb +++ b/services/QuillLMS/engines/evidence/spec/models/evidence/research/gen_ai/trial_spec.rb @@ -174,6 +174,20 @@ module GenAI end end + describe '#set_feedback_evaluation_metrics' do + let(:llm_example1) { create(:evidence_research_gen_ai_llm_example, trial:) } + let(:llm_example2) { create(:evidence_research_gen_ai_llm_example, trial:) } + let!(:feedback_evalution1) { create(:evidence_research_gen_ai_feedback_evaluation, :optimal, trial:, llm_example: llm_example1) } + let!(:feedback_evalution2) { create(:evidence_research_gen_ai_feedback_evaluation, :suboptimal, trial:, llm_example: llm_example2) } + + it 'sets feedback_failure_count and feedback_failure_percentage' do + trial.set_feedback_evaluation_metrics + + expect(trial.feedback_failure_count).to eq 1 + expect(trial.feedback_failure_percentage).to eq 0.5 + end + end + describe '#update_results!' do subject { trial.update_results!(new_data) } diff --git a/services/QuillLMS/engines/evidence/spec/services/evidence/gen_ai/evaluation/creator_spec.rb b/services/QuillLMS/engines/evidence/spec/services/evidence/gen_ai/evaluation/creator_spec.rb index 7cce20f4ea0..38c3ef53bc3 100644 --- a/services/QuillLMS/engines/evidence/spec/services/evidence/gen_ai/evaluation/creator_spec.rb +++ b/services/QuillLMS/engines/evidence/spec/services/evidence/gen_ai/evaluation/creator_spec.rb @@ -31,7 +31,7 @@ it 'creates a FeedbackEvaluation record' do expect(Evidence::Research::GenAI::FeedbackEvaluation).to receive(:create!).with( trial_id:, - version: , + version:, llm_example_id:, optimal: true, errored_checks: [] diff --git a/services/QuillLMS/engines/evidence/spec/workers/evidence/research/gen_ai/calculate_results_worker_spec.rb b/services/QuillLMS/engines/evidence/spec/workers/evidence/research/gen_ai/calculate_results_worker_spec.rb index 646c432fe45..2beebeb50a6 100644 --- a/services/QuillLMS/engines/evidence/spec/workers/evidence/research/gen_ai/calculate_results_worker_spec.rb +++ b/services/QuillLMS/engines/evidence/spec/workers/evidence/research/gen_ai/calculate_results_worker_spec.rb @@ -31,6 +31,7 @@ module GenAI allow(trial).to receive(:set_confusion_matrix) allow(trial).to receive(:set_evaluation_start_time) allow(GEvalScoresFetcher).to receive(:run).with(trial) + allow(RunTrialEvaluationWorker).to receive(:perform_async).with(trial.id) end it 'sets the confusion matrix' do @@ -45,6 +46,7 @@ module GenAI it 'runs the GEvalScoresFetcher' do expect(GEvalScoresFetcher).to receive(:run).with(trial) + expect(RunTrialEvaluationWorker).to receive(:perform_async).with(trial.id) subject end end diff --git a/services/QuillLMS/engines/evidence/spec/workers/evidence/research/gen_ai/run_feedback_evaluation_worker_spec.rb b/services/QuillLMS/engines/evidence/spec/workers/evidence/research/gen_ai/run_feedback_evaluation_worker_spec.rb new file mode 100644 index 00000000000..f3b538ba16a --- /dev/null +++ b/services/QuillLMS/engines/evidence/spec/workers/evidence/research/gen_ai/run_feedback_evaluation_worker_spec.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +require 'rails_helper' +require 'sidekiq/testing' + +RSpec.describe Evidence::Research::GenAI::RunFeedbackEvaluationWorker, type: :worker do + let(:trial_id) { 1 } + let(:version) { 1 } + let(:llm_example_id) { 100 } + + describe '#perform' do + subject { described_class.new.perform(trial_id, version, llm_example_id) } + + it 'calls Evidence::GenAI::Evaluation::Creator with the correct arguments' do + expect(Evidence::GenAI::Evaluation::Creator).to receive(:run).with( + trial_id:, + version:, + llm_example_id: + ) + + subject + end + end +end diff --git a/services/QuillLMS/engines/evidence/spec/workers/evidence/research/gen_ai/run_trial_evaluation_worker_spec.rb b/services/QuillLMS/engines/evidence/spec/workers/evidence/research/gen_ai/run_trial_evaluation_worker_spec.rb new file mode 100644 index 00000000000..9904f54a338 --- /dev/null +++ b/services/QuillLMS/engines/evidence/spec/workers/evidence/research/gen_ai/run_trial_evaluation_worker_spec.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +require 'rails_helper' +require 'sidekiq/testing' + +RSpec.describe Evidence::Research::GenAI::RunTrialEvaluationWorker, type: :worker do + let(:trial) { create(:evidence_research_gen_ai_trial) } + let(:version) { described_class::VERSION } + let!(:suboptimal_example1) { create(:evidence_research_gen_ai_llm_example, :suboptimal, trial: ) } + let!(:suboptimal_example2) { create(:evidence_research_gen_ai_llm_example, :suboptimal, trial:) } + let!(:optimal_example1) { create(:evidence_research_gen_ai_llm_example, :optimal, trial:) } + + describe '#perform' do + subject { described_class.new.perform(trial.id) } + + it 'enqueues feedback evaluation jobs for suboptimal examples' do + expect(Evidence::Research::GenAI::RunFeedbackEvaluationWorker).to receive(:perform_async) + .with(trial.id, version, suboptimal_example1.id) + expect(Evidence::Research::GenAI::RunFeedbackEvaluationWorker).to receive(:perform_async) + .with(trial.id, version, suboptimal_example2.id) + + subject + end + end + + describe '#on_complete' do + let(:options) { { 'trial_id' => trial.id } } + + subject { described_class.new.on_complete(nil, options) } + + it 'sets feedback failure metrics' do + expect_any_instance_of(Evidence::Research::GenAI::Trial).to receive(:set_feedback_evaluation_metrics) + + subject + end + end +end From d2f630801151bbf7013ec738c15cdcbf9af60ccf Mon Sep 17 00:00:00 2001 From: Daniel Drabik Date: Fri, 31 Jan 2025 14:32:38 -0500 Subject: [PATCH 34/51] Self review cleanup. --- services/QuillLMS/db/structure.sql | 2 +- .../QuillLMS/engines/evidence/README.rdoc | 4 +-- .../gen_ai/evaluation/data_fetcher.rb | 1 - .../gen_ai/evaluation/verbose/checker.rb | 1 - .../research/gen_ai/feedback_evaluations.rb | 29 +++++++++++++++---- .../run_trial_evaluation_worker_spec.rb | 4 +-- 6 files changed, 29 insertions(+), 12 deletions(-) diff --git a/services/QuillLMS/db/structure.sql b/services/QuillLMS/db/structure.sql index 18ce70c372a..6015c9298ab 100644 --- a/services/QuillLMS/db/structure.sql +++ b/services/QuillLMS/db/structure.sql @@ -12072,7 +12072,7 @@ ALTER TABLE ONLY public.learn_worlds_account_course_events -- PostgreSQL database dump complete -- -SET search_path TO "$user", public, heroku_ext; +SET search_path TO "$user", public; INSERT INTO "schema_migrations" (version) VALUES ('20250130191426'), diff --git a/services/QuillLMS/engines/evidence/README.rdoc b/services/QuillLMS/engines/evidence/README.rdoc index fe5a934b082..107def095f7 100644 --- a/services/QuillLMS/engines/evidence/README.rdoc +++ b/services/QuillLMS/engines/evidence/README.rdoc @@ -47,10 +47,10 @@ After generating the scaffold You run migrations in the parent app, not in the engine. So if you add a migration, run this in the parent app: -`bin/rails evidence:install:migrations` +`rake evidence:install:migrations` If you are hitting an issue in which running `rspec` keeps telling you have a pending migration (and you've run the LMS migrations), run the following in the evidence directory: -`bundle exec rake db:schema:load RAILS_ENV=test` +`rake db:schema:load RAILS_ENV=test` See the Rails guide for more details. diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/data_fetcher.rb b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/data_fetcher.rb index 1f5b8baf640..de93beab2b2 100644 --- a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/data_fetcher.rb +++ b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/data_fetcher.rb @@ -12,7 +12,6 @@ class DataFetcher < ApplicationService HAITIAN_BECAUSE = 'new_haitian_because.csv' DEFAULT_FILE = WHALES_SO - CSV_FILE_PATH = "#{Evidence::Engine.root}/app/services/evidence/gen_ai/evaluation/data/%s" attr_reader :rule, :optimal diff --git a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/verbose/checker.rb b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/verbose/checker.rb index 37bcb76ad73..c764e864cbe 100644 --- a/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/verbose/checker.rb +++ b/services/QuillLMS/engines/evidence/app/services/evidence/gen_ai/evaluation/verbose/checker.rb @@ -6,7 +6,6 @@ module Evaluation module Verbose class Checker < Evaluation::BaseChecker MAX_SIZE = 325 - MAX_SENTENCES = 4 def run diff --git a/services/QuillLMS/engines/evidence/spec/factories/evidence/research/gen_ai/feedback_evaluations.rb b/services/QuillLMS/engines/evidence/spec/factories/evidence/research/gen_ai/feedback_evaluations.rb index b836abf19b0..99618d9cacb 100644 --- a/services/QuillLMS/engines/evidence/spec/factories/evidence/research/gen_ai/feedback_evaluations.rb +++ b/services/QuillLMS/engines/evidence/spec/factories/evidence/research/gen_ai/feedback_evaluations.rb @@ -1,4 +1,23 @@ +# frozen_string_literal: true +# == Schema Information +# +# Table name: evidence_research_gen_ai_feedback_evaluations +# +# id :bigint not null, primary key +# errored_checks :string default([]), is an Array +# optimal :boolean not null +# version :integer not null +# created_at :datetime not null +# updated_at :datetime not null +# llm_example_id :integer not null +# trial_id :integer not null +# +# Indexes +# +# idx_on_llm_example_id_774518eafd (llm_example_id) +# idx_on_trial_id_eee55ce93c (trial_id) +# module Evidence module Research module GenAI @@ -7,13 +26,13 @@ module GenAI version { 1 } llm_example { association :evidence_research_gen_ai_llm_example } trial { association :evidence_research_gen_ai_trial } - optimal true - errored_checks [] + optimal { true } + errored_checks { [] } - trait(:optimal) { optimal true } + trait(:optimal) { optimal { true } } trait(:suboptimal) do - optimal false - errored_checks ['tone'] + optimal { false } + errored_checks { ['tone'] } end end end diff --git a/services/QuillLMS/engines/evidence/spec/workers/evidence/research/gen_ai/run_trial_evaluation_worker_spec.rb b/services/QuillLMS/engines/evidence/spec/workers/evidence/research/gen_ai/run_trial_evaluation_worker_spec.rb index 9904f54a338..8f557ff17ec 100644 --- a/services/QuillLMS/engines/evidence/spec/workers/evidence/research/gen_ai/run_trial_evaluation_worker_spec.rb +++ b/services/QuillLMS/engines/evidence/spec/workers/evidence/research/gen_ai/run_trial_evaluation_worker_spec.rb @@ -6,7 +6,7 @@ RSpec.describe Evidence::Research::GenAI::RunTrialEvaluationWorker, type: :worker do let(:trial) { create(:evidence_research_gen_ai_trial) } let(:version) { described_class::VERSION } - let!(:suboptimal_example1) { create(:evidence_research_gen_ai_llm_example, :suboptimal, trial: ) } + let!(:suboptimal_example1) { create(:evidence_research_gen_ai_llm_example, :suboptimal, trial:) } let!(:suboptimal_example2) { create(:evidence_research_gen_ai_llm_example, :suboptimal, trial:) } let!(:optimal_example1) { create(:evidence_research_gen_ai_llm_example, :optimal, trial:) } @@ -26,7 +26,7 @@ describe '#on_complete' do let(:options) { { 'trial_id' => trial.id } } - subject { described_class.new.on_complete(nil, options) } + subject { described_class.new.on_complete(nil, options) } it 'sets feedback failure metrics' do expect_any_instance_of(Evidence::Research::GenAI::Trial).to receive(:set_feedback_evaluation_metrics) From a896d1e19db857b766204af38237158f525e7d91 Mon Sep 17 00:00:00 2001 From: Daniel Drabik Date: Fri, 31 Jan 2025 14:40:23 -0500 Subject: [PATCH 35/51] Clean up schema. --- services/QuillLMS/db/structure.sql | 1 - 1 file changed, 1 deletion(-) diff --git a/services/QuillLMS/db/structure.sql b/services/QuillLMS/db/structure.sql index 6015c9298ab..8b2a6e06176 100644 --- a/services/QuillLMS/db/structure.sql +++ b/services/QuillLMS/db/structure.sql @@ -12076,7 +12076,6 @@ SET search_path TO "$user", public; INSERT INTO "schema_migrations" (version) VALUES ('20250130191426'), -('20250109202521'), ('20250109151117'), ('20250109202521'), ('20241211155124'), From 4121cf2638401f1ea4a0ef51125a07a1944f9d23 Mon Sep 17 00:00:00 2001 From: Daniel Drabik Date: Fri, 31 Jan 2025 15:16:27 -0500 Subject: [PATCH 36/51] Rename method. --- .../app/models/evidence/research/gen_ai/trial.rb | 14 ++++++-------- .../research/gen_ai/run_trial_evaluation_worker.rb | 2 +- .../models/evidence/research/gen_ai/trial_spec.rb | 8 ++++---- .../gen_ai/run_trial_evaluation_worker_spec.rb | 4 ++-- 4 files changed, 13 insertions(+), 15 deletions(-) diff --git a/services/QuillLMS/engines/evidence/app/models/evidence/research/gen_ai/trial.rb b/services/QuillLMS/engines/evidence/app/models/evidence/research/gen_ai/trial.rb index 63c9fb48935..755031c1f8c 100644 --- a/services/QuillLMS/engines/evidence/app/models/evidence/research/gen_ai/trial.rb +++ b/services/QuillLMS/engines/evidence/app/models/evidence/research/gen_ai/trial.rb @@ -72,8 +72,8 @@ class Trial < ApplicationRecord :optimal_accuracy, :suboptimal_accuracy, :weighted_accuracy, - :feedback_failure_count, - :feedback_failure_percentage + :evaluator_failure_count, + :evaluator_total_count attr_readonly :llm_id, :llm_prompt_id, :dataset_id, :temperature @@ -122,13 +122,11 @@ def set_trial_start_time = update!(trial_start_time: Time.zone.now) def set_trial_duration = update!(trial_duration: Time.zone.now - Time.zone.parse(trial_start_time)) def set_status = trial_errors.empty? ? completed! : failed! - def set_feedback_evaluation_metrics - feedback_failure_count = feedback_evaluations.suboptimal.count - total = feedback_evaluations.count + def set_evaluator_counts + evaluator_failure_count = feedback_evaluations.suboptimal.count + evaluator_total_count = feedback_evaluations.count - feedback_failure_percentage = (feedback_failure_count / total.to_f).round(4) - - update(feedback_failure_count:, feedback_failure_percentage:) + update(evaluator_failure_count:, evaluator_total_count:) end private def set_accuracy diff --git a/services/QuillLMS/engines/evidence/app/workers/evidence/research/gen_ai/run_trial_evaluation_worker.rb b/services/QuillLMS/engines/evidence/app/workers/evidence/research/gen_ai/run_trial_evaluation_worker.rb index 7eebb2fa497..03feb5a63bd 100644 --- a/services/QuillLMS/engines/evidence/app/workers/evidence/research/gen_ai/run_trial_evaluation_worker.rb +++ b/services/QuillLMS/engines/evidence/app/workers/evidence/research/gen_ai/run_trial_evaluation_worker.rb @@ -27,7 +27,7 @@ def perform(trial_id) def on_complete(_status, options) Trial .find(options['trial_id']) - .set_feedback_evaluation_metrics + .set_evaluator_counts end end end diff --git a/services/QuillLMS/engines/evidence/spec/models/evidence/research/gen_ai/trial_spec.rb b/services/QuillLMS/engines/evidence/spec/models/evidence/research/gen_ai/trial_spec.rb index c2ccb81d351..1301e958a19 100644 --- a/services/QuillLMS/engines/evidence/spec/models/evidence/research/gen_ai/trial_spec.rb +++ b/services/QuillLMS/engines/evidence/spec/models/evidence/research/gen_ai/trial_spec.rb @@ -174,17 +174,17 @@ module GenAI end end - describe '#set_feedback_evaluation_metrics' do + describe '#set_evaluator_counts' do let(:llm_example1) { create(:evidence_research_gen_ai_llm_example, trial:) } let(:llm_example2) { create(:evidence_research_gen_ai_llm_example, trial:) } let!(:feedback_evalution1) { create(:evidence_research_gen_ai_feedback_evaluation, :optimal, trial:, llm_example: llm_example1) } let!(:feedback_evalution2) { create(:evidence_research_gen_ai_feedback_evaluation, :suboptimal, trial:, llm_example: llm_example2) } it 'sets feedback_failure_count and feedback_failure_percentage' do - trial.set_feedback_evaluation_metrics + trial.set_evaluator_counts - expect(trial.feedback_failure_count).to eq 1 - expect(trial.feedback_failure_percentage).to eq 0.5 + expect(trial.evaluator_failure_count).to eq 1 + expect(trial.evaluator_total_count).to eq 2 end end diff --git a/services/QuillLMS/engines/evidence/spec/workers/evidence/research/gen_ai/run_trial_evaluation_worker_spec.rb b/services/QuillLMS/engines/evidence/spec/workers/evidence/research/gen_ai/run_trial_evaluation_worker_spec.rb index 8f557ff17ec..609ab2b1091 100644 --- a/services/QuillLMS/engines/evidence/spec/workers/evidence/research/gen_ai/run_trial_evaluation_worker_spec.rb +++ b/services/QuillLMS/engines/evidence/spec/workers/evidence/research/gen_ai/run_trial_evaluation_worker_spec.rb @@ -28,8 +28,8 @@ subject { described_class.new.on_complete(nil, options) } - it 'sets feedback failure metrics' do - expect_any_instance_of(Evidence::Research::GenAI::Trial).to receive(:set_feedback_evaluation_metrics) + it 'sets evaluator counts' do + expect_any_instance_of(Evidence::Research::GenAI::Trial).to receive(:set_evaluator_counts) subject end From fc2ac784702d2d27dbe6f41d32120dce3c09bce3 Mon Sep 17 00:00:00 2001 From: Daniel Drabik Date: Fri, 31 Jan 2025 16:43:49 -0500 Subject: [PATCH 37/51] Working version of error stats on trial index view. --- .../individualDataset/trialsSection.tsx | 11 ++++++++++- .../bundles/Staff/interfaces/evidenceInterfaces.ts | 2 ++ .../app/models/evidence/research/gen_ai/trial.rb | 2 +- .../evidence/research/gen_ai/datasets/show.html.erb | 2 ++ 4 files changed, 15 insertions(+), 2 deletions(-) diff --git a/services/QuillLMS/client/app/bundles/Staff/components/evidence/llmPromptDatasets/individualDataset/trialsSection.tsx b/services/QuillLMS/client/app/bundles/Staff/components/evidence/llmPromptDatasets/individualDataset/trialsSection.tsx index fbdc8f4bbf1..3e7a2c516ff 100644 --- a/services/QuillLMS/client/app/bundles/Staff/components/evidence/llmPromptDatasets/individualDataset/trialsSection.tsx +++ b/services/QuillLMS/client/app/bundles/Staff/components/evidence/llmPromptDatasets/individualDataset/trialsSection.tsx @@ -104,6 +104,14 @@ const TrialsSection = ({ trials, datasetPath, }: { trials: TrialInterface[], dat rowSectionClassName: 'center-content allow-wrap', noTooltip: true }, + { + name: 'Feedback Error Rate', + attribute: 'feedbackErrorRate', + width: '64px', + headerClassName: 'center-content', + rowSectionClassName: 'center-content allow-wrap', + noTooltip: true + }, { name: 'LLM', attribute: 'llmVersion', @@ -151,7 +159,7 @@ const TrialsSection = ({ trials, datasetPath, }: { trials: TrialInterface[], dat ] const rows = () => trials.map(trial => { - const { number, created_at, temperature, optimal_correct, optimal_count, suboptimal_correct, suboptimal_count, average_g_eval_score, status, id, notes, llm_version, llm_prompt, } = trial + const { number, created_at, temperature, optimal_correct, optimal_count, suboptimal_correct, suboptimal_count, average_g_eval_score, status, id, notes, llm_version, llm_prompt, evaluator_failure_count, evaluator_total_count } = trial const { name, optimal_examples_count, suboptimal_examples_count, guidelines_count, } = llm_prompt let compareCheckbox =