From 8ba68f973810848f455334363fb8c4612dc6803b Mon Sep 17 00:00:00 2001 From: Paul Abumov Date: Thu, 17 Aug 2023 12:02:58 -0400 Subject: [PATCH] Updated Prolific integrations docs --- .../abstractions/providers/prolific/README.md | 145 +++++++++++- .../providers/prolific/api/README.md | 224 ++++++++++++------ .../providers/prolific/prolific_utils.py | 57 ++++- 3 files changed, 345 insertions(+), 81 deletions(-) diff --git a/mephisto/abstractions/providers/prolific/README.md b/mephisto/abstractions/providers/prolific/README.md index 23c4b403b..2a4bc12ae 100644 --- a/mephisto/abstractions/providers/prolific/README.md +++ b/mephisto/abstractions/providers/prolific/README.md @@ -1,5 +1,57 @@ ### Using Eligibility Requirements in configuring TaskRun +Prolific Eligibility Requirements are the equivalent of Mephisto Qualifications, +and are used to allow Participants to see our published Studies in Prolific. + +To enable Participant filtering by custom Mephisto Qualifications +(i.e. not supported by Prolific), we create a one-time (dynamically updated) Participant Group +containing all known Participants filtered by those custom qualifications. +Then we pass ID of that Eligibility Participant Group (EPG) to Prolific API via +`ParticipantGroupEligibilityRequirement` Study requirement. After Study concludes, this +disposable EPG is removed. + +NOTE: If you use several Eligibility Requirements together (e.g. +`CustomBlacklistEligibilityRequirement`, `CustomWhitelistEligibilityRequirement`, and +`ParticipantGroupEligibilityRequirement`), then Participants will be selected by +intersecting all of these requirements. + + +------------------------------------------------------------------------------- +#### Usage + +Shared state handles both custom and Prolific-supported qualifications: + +```python +shared_state.prolific_specific_qualifications = [ + { + 'name': 'AgeRangeEligibilityRequirement', + 'min_age': 18, + 'max_age': 100, + }, +] + +shared_state.qualifications = [ + make_qualification_dict('sample_qual_name', QUAL_GREATER_EQUAL, 1), +] +``` + +You can alternatively indicate Prolific-supported requirements in Hydra YAML config +under the key `prolific_eligibility_requirements` like so: + +```yaml +mephisto: + provider: + prolific_eligibility_requirements: + - name: 'AgeRangeEligibilityRequirement' + min_age: 10 + max_age: 20 +``` + + + +------------------------------------------------------------------------------- +#### Available Eligibility Requirements + List of supported Eligibility Requirements for `SharedState.prolific_specific_qualifications`: ```python @@ -41,14 +93,87 @@ List of supported Eligibility Requirements for `SharedState.prolific_specific_qu _NOTE: this list (last updated 2023.07.11) is derived from list of available classes in `mephisto/abstractions/providers/prolific/api/eligibility_requirement_classes`._ -You can use these requirements in Hydra YAML config under the key -`prolific_eligibility_requirements` like so: -```yaml -mephisto: - provider: - prolific_eligibility_requirements: - - name: 'AgeRangeEligibilityRequirement' - min_age: 10 - max_age: 20 -``` + +------------------------------------------------------------------------------- +#### Block lists and Allow lists + +Currently we're not using Participant Groups called `prolific_allow_list_group_name` +and `prolific_block_list_group_name` for anything (even though we diligently maintain them +via Prolific's API). + +- Manual allow list is applied via `CustomWhitelistEligibilityRequirement` +- Manual block list is applied via `CustomBlacklistEligibilityRequirement`. +- Automatic block list is applied by not including ineligible Participants into EPG +as per `is_blocked` column in the prolific datastore table. + + + +------------------------------------------------------------------------------- +### Prolific API oddities + +#### Statuses + +Prolific lacks several important statuses for Studies: + +- `EXPIRED` status: we simulate it with adding a special suffix to `Study.internal_name`. +- `STOPPED` status: whether Study has naturally completed, +or it was manually stopped via Prolific UI, in both cases it will transition to `AWAITING_REVIEW` +status. +- `FINISHED` status: all `COMPLETED` and `AWAITING_REVIEW` studies in Prolific are not really +completed - they can still be reactivated by increasing their `total_available_places` parameter. + +You may also bump against some unobvious behaviours of Prolific Studies: + +1. We cannot stop Study via Prolific UI when `max_num_concurrent_units` is specified, +because Mephisto will keep on launching additional Units (and thus reactivating the Study) +every time Study falls into `AWAITING_REVIEW` status, until all created Units have been launched. +2. When we use `max_num_concurrent_units`, every time available places drop to zero for a moment, +Prolific will automatically transition Study into `AWAITING_REVIEW` status, +as if the Study has already finished. +3. Every time Prolific is updating status of a Study or Submission, it briefly puts that object +status to an undocumented `PROCESSING` status. + + + +------------------------------------------------------------------------------- +#### Studies/Submissions + +1. When Prolific Study is created with `ParticipantGroupEligibilityRequirement`, +Prolific UI doesn't show it under `Study.eligibility_requirements` - it's only +visible via an API request. (And `id` in that Eligibility Requirement will be +not the EPG ID, but rather some internal ID from Prolific DB.) +2. After Study has been publiched, Prolific only allows updating `internal_name` and +`total_available_places` (all other fields are read-only, including Allow List and Block List). +This is the reason we're using disposable EPGs for every Study. +3. Prolific only allows increasing (but not decreasing) `total_available_places` +4. The way to set Submission as completed is not by issuing an API call, but by redirecting +a Participant in their browser to a page with a specific URL that includes a Study completion code. +5. Prolific does not allow to specify custom Study completion code during creation of a Study, +therefore we have to generate our own completion codes based on Study's ID and +update it in Prolific when creating a Study. + + + +------------------------------------------------------------------------------- +#### Groups + +1. Prolific caches `eligible_participant_count` for Groups (so it always shows the initial value, +despite any changes of Particpants in the Group) +2. Deletion of Participant Groups is soft on Prolific side, and their API response doesn't +indicate that a Group has been soft-deleted (as if it's still active) + + + +------------------------------------------------------------------------------- +### Steps of running a Study + +1. Create inactive Study during TaskRun launch, set its `total_available_places = None`. +(Mephisto will create all required units with `created` status.) +2. Update Study with generated `completion_codes` (we need `Study.id` for it). +3. Publish the Study on Prolific. +4. When Mephisto launches a new Unit, increase `total_available_places` in Prolific Study. +5. When a Unit is expired (i.e. completed or returned), Mephisto launches another Unit. +6. When all Units are expired, the Study is considered completed. +7. The results are reviewed by a user. (All currently active EPGs are dynamically changed, +based on workers' updated custom qualifications.) diff --git a/mephisto/abstractions/providers/prolific/api/README.md b/mephisto/abstractions/providers/prolific/api/README.md index d55ea1be4..7845de745 100644 --- a/mephisto/abstractions/providers/prolific/api/README.md +++ b/mephisto/abstractions/providers/prolific/api/README.md @@ -1,9 +1,8 @@ ## Prolific Python SDK -------------------------------------------------------------------------------- - +------------------------------------------------------------------------------- ### Usage Set the API key as an environment variable: @@ -36,10 +35,11 @@ look at `mephisto.abstractions.providers.prolific.api.data_models.User` #### Retrieve user account info ```python -from mephisto.abstractions.providers.prolific import api as prolific_api from mephisto.abstractions.providers.prolific.api.data_models import User +from mephisto.abstractions.providers.prolific.prolific_utils import get_authenticated_client -user: User = prolific_api.Users.me() +client = get_authenticated_client("prolific") +user: User = client.Users.me() ``` @@ -53,33 +53,58 @@ look at `mephisto.abstractions.providers.prolific.api.data_models.Workspace` #### List ```python -from mephisto.abstractions.providers.prolific import api as prolific_api from mephisto.abstractions.providers.prolific.api.data_models import Workspace +from mephisto.abstractions.providers.prolific.prolific_utils import get_authenticated_client -workspace_list: List[Workspace] = prolific_api.Workspaces.list() +client = get_authenticated_client("prolific") +workspace_list: List[Workspace] = client.Workspaces.list() ``` #### Retrieve ```python -from mephisto.abstractions.providers.prolific import api as prolific_api from mephisto.abstractions.providers.prolific.api.data_models import Workspace +from mephisto.abstractions.providers.prolific.prolific_utils import get_authenticated_client -workspace: Workspace = prolific_api.Workspaces.retrieve(id='XXXXXXXXXXXXXXXXXXXXXXXX') +client = get_authenticated_client("prolific") +workspace: Workspace = client.Workspaces.retrieve(id='XXXXXXXXXXXXXXXXXXXXXXXX') ``` #### Create ```python -from mephisto.abstractions.providers.prolific import api as prolific_api from mephisto.abstractions.providers.prolific.api.data_models import Workspace +from mephisto.abstractions.providers.prolific.prolific_utils import get_authenticated_client -workspace: Workspace = prolific_api.Workspaces.create( +client = get_authenticated_client("prolific") +workspace: Workspace = client.Workspaces.create( title='Title', description='Description', ) ``` +#### Update + +```python +from mephisto.abstractions.providers.prolific.api.data_models import Workspace +from mephisto.abstractions.providers.prolific.prolific_utils import get_authenticated_client + +client = get_authenticated_client("prolific") +workspace: Workspace = client.Workspaces.update( + id='XXXXXXXXXXXXXXXXXXXXXXXX', + description='Description', +) +``` + +#### Get balance + +```python +from mephisto.abstractions.providers.prolific.api.data_models import WorkspaceBalance +from mephisto.abstractions.providers.prolific.prolific_utils import get_authenticated_client + +client = get_authenticated_client("prolific") +workspace: WorkspaceBalance = client.Workspaces.get_balance(id='XXXXXXXXXXXXXXXXXXXXXXXX') +``` ------------------------------------------------------------------------------- @@ -91,10 +116,11 @@ look at `mephisto.abstractions.providers.prolific.api.data_models.Project` #### List for Workspace ```python -from mephisto.abstractions.providers.prolific import api as prolific_api from mephisto.abstractions.providers.prolific.api.data_models import Project +from mephisto.abstractions.providers.prolific.prolific_utils import get_authenticated_client -project_list: List[Project] = prolific_api.Projects.list_for_workspace( +client = get_authenticated_client("prolific") +project_list: List[Project] = client.Projects.list_for_workspace( workspace_id='XXXXXXXXXXXXXXXXXXXXXXXX', ) ``` @@ -102,10 +128,11 @@ project_list: List[Project] = prolific_api.Projects.list_for_workspace( #### Retrieve for Workspace ```python -from mephisto.abstractions.providers.prolific import api as prolific_api from mephisto.abstractions.providers.prolific.api.data_models import Project +from mephisto.abstractions.providers.prolific.prolific_utils import get_authenticated_client -project: Project = prolific_api.Projects.retrieve_for_workspace( +client = get_authenticated_client("prolific") +project: Project = client.Projects.retrieve_for_workspace( workspace_id='XXXXXXXXXXXXXXXXXXXXXXXX', project_id='YYYYYYYYYYYYYYYYYYYYYYYYY', ) @@ -114,10 +141,11 @@ project: Project = prolific_api.Projects.retrieve_for_workspace( #### Create for Workspace ```python -from mephisto.abstractions.providers.prolific import api as prolific_api from mephisto.abstractions.providers.prolific.api.data_models import Project +from mephisto.abstractions.providers.prolific.prolific_utils import get_authenticated_client -project: Project = prolific_api.Projects.create_for_workspace( +client = get_authenticated_client("prolific") +project: Project = client.Projects.create_for_workspace( workspace_id='XXXXXXXXXXXXXXXXXXXXXXXX', title='Title', ) @@ -134,19 +162,21 @@ look at `mephisto.abstractions.providers.prolific.api.data_models.Study` #### List ```python -from mephisto.abstractions.providers.prolific import api as prolific_api from mephisto.abstractions.providers.prolific.api.data_models import Study +from mephisto.abstractions.providers.prolific.prolific_utils import get_authenticated_client -study_list: List[Study] = prolific_api.Studies.list() +client = get_authenticated_client("prolific") +study_list: List[Study] = client.Studies.list() ``` #### List for Project ```python -from mephisto.abstractions.providers.prolific import api as prolific_api from mephisto.abstractions.providers.prolific.api.data_models import Study +from mephisto.abstractions.providers.prolific.prolific_utils import get_authenticated_client -study_list: List[Study] = prolific_api.Studies.list_for_project( +client = get_authenticated_client("prolific") +study_list: List[Study] = client.Studies.list_for_project( project_id='YYYYYYYYYYYYYYYYYYYYYYYYY', ) ``` @@ -154,19 +184,21 @@ study_list: List[Study] = prolific_api.Studies.list_for_project( #### Retrieve ```python -from mephisto.abstractions.providers.prolific import api as prolific_api from mephisto.abstractions.providers.prolific.api.data_models import Study +from mephisto.abstractions.providers.prolific.prolific_utils import get_authenticated_client -study: Study = prolific_api.Studies.retrieve(id='XXXXXXXXXXXXXXXXXXXXXXXX') +client = get_authenticated_client("prolific") +study: Study = client.Studies.retrieve(id='XXXXXXXXXXXXXXXXXXXXXXXX') ``` #### Create ```python -from mephisto.abstractions.providers.prolific import api as prolific_api from mephisto.abstractions.providers.prolific.api.data_models import Study +from mephisto.abstractions.providers.prolific.prolific_utils import get_authenticated_client -study: Study = prolific_api.Studies.create( +client = get_authenticated_client("prolific") +study: Study = client.Studies.create( project='YYYYYYYYYYYYYYYYYYYYYYYYY', name='Name', internal_name='Internal name', @@ -191,10 +223,11 @@ study: Study = prolific_api.Studies.create( #### Update ```python -from mephisto.abstractions.providers.prolific import api as prolific_api from mephisto.abstractions.providers.prolific.api.data_models import Study +from mephisto.abstractions.providers.prolific.prolific_utils import get_authenticated_client -study: Study = prolific_api.Studies.update( +client = get_authenticated_client("prolific") +study: Study = client.Studies.update( id='XXXXXXXXXXXXXXXXXXXXXXXX', name='Name', internal_name='Internal name', @@ -205,26 +238,39 @@ study: Study = prolific_api.Studies.update( #### Remove ```python -from mephisto.abstractions.providers.prolific import api as prolific_api +from mephisto.abstractions.providers.prolific.prolific_utils import get_authenticated_client -study = prolific_api.Studies.remove(id='XXXXXXXXXXXXXXXXXXXXXXXX') +client = get_authenticated_client("prolific") +study = client.Studies.remove(id='XXXXXXXXXXXXXXXXXXXXXXXX') ``` #### Publish ```python -from mephisto.abstractions.providers.prolific import api as prolific_api from mephisto.abstractions.providers.prolific.api.data_models import Study +from mephisto.abstractions.providers.prolific.prolific_utils import get_authenticated_client -study: Study = prolific_api.Studies.publish(id='XXXXXXXXXXXXXXXXXXXXXXXX') +client = get_authenticated_client("prolific") +study: Study = client.Studies.publish(id='XXXXXXXXXXXXXXXXXXXXXXXX') +``` + +#### Stop + +```python +from mephisto.abstractions.providers.prolific.api.data_models import Study +from mephisto.abstractions.providers.prolific.prolific_utils import get_authenticated_client + +client = get_authenticated_client("prolific") +study: Study = client.Studies.stop(id='XXXXXXXXXXXXXXXXXXXXXXXX') ``` #### Calculate Cost ```python -from mephisto.abstractions.providers.prolific import api as prolific_api +from mephisto.abstractions.providers.prolific.prolific_utils import get_authenticated_client -study: Union[int, float] = prolific_api.Studies.calculate_cost(reward=100, total_available_places=2) +client = get_authenticated_client("prolific") +study: Union[int, float] = client.Studies.calculate_cost(reward=100, total_available_places=2) ``` @@ -238,35 +284,59 @@ look at `mephisto.abstractions.providers.prolific.api.data_models.ParticipantGro #### List ```python -from mephisto.abstractions.providers.prolific import api as prolific_api from mephisto.abstractions.providers.prolific.api.data_models import ParticipantGroup +from mephisto.abstractions.providers.prolific.prolific_utils import get_authenticated_client -participan_group_list: List[ParticipantGroup] = prolific_api.ParticipantGroups.list() +client = get_authenticated_client("prolific") +participan_group_list: List[ParticipantGroup] = client.ParticipantGroups.list() # Or for Project -participan_group_list: List[ParticipantGroup] = prolific_api.ParticipantGroups.list( +participan_group_list: List[ParticipantGroup] = client.ParticipantGroups.list( project_id='YYYYYYYYYYYYYYYYYYYYYYYYY', ) ``` +#### Retrieve + +```python +from mephisto.abstractions.providers.prolific.api.data_models import ParticipantGroup +from mephisto.abstractions.providers.prolific.prolific_utils import get_authenticated_client + +client = get_authenticated_client("prolific") +participan_group: ParticipantGroup = client.ParticipantGroups.retrieve( + id='PPPPPPPPPPPPPPPPPPPPPPPPP', +) +``` + #### Create ```python -from mephisto.abstractions.providers.prolific import api as prolific_api from mephisto.abstractions.providers.prolific.api.data_models import ParticipantGroup +from mephisto.abstractions.providers.prolific.prolific_utils import get_authenticated_client -participan_group: ParticipantGroup = prolific_api.ParticipantGroups.create( +client = get_authenticated_client("prolific") +participan_group: ParticipantGroup = client.ParticipantGroups.create( project_id='YYYYYYYYYYYYYYYYYYYYYYYYY', name='Name', ) ``` +#### Remove + +```python +from mephisto.abstractions.providers.prolific.prolific_utils import get_authenticated_client + +client = get_authenticated_client("prolific") +client.ParticipantGroups.remove(id='PPPPPPPPPPPPPPPPPPPPPPPPP') +``` + #### List of Perticipants for Participant Group ```python -from mephisto.abstractions.providers.prolific import api as prolific_api from mephisto.abstractions.providers.prolific.api.data_models import Participant +from mephisto.abstractions.providers.prolific.prolific_utils import get_authenticated_client -participant_list: List[Participant] = prolific_api.ParticipantGroups.list_perticipants_for_group( +client = get_authenticated_client("prolific") +participant_list: List[Participant] = client.ParticipantGroups.list_perticipants_for_group( id='PPPPPPPPPPPPPPPPPPPPPPPPP', ) ``` @@ -274,11 +344,12 @@ participant_list: List[Participant] = prolific_api.ParticipantGroups.list_pertic #### Add Perticipants to Participant Group ```python -from mephisto.abstractions.providers.prolific import api as prolific_api from mephisto.abstractions.providers.prolific.api.data_models import Participant +from mephisto.abstractions.providers.prolific.prolific_utils import get_authenticated_client -participant_list: List[Participant] = prolific_api.ParticipantGroups.add_perticipants_to_group( - id='PPPPPPPPPPPPPPPPPPPPPPPPP', +client = get_authenticated_client("prolific") +participant_list: List[Participant] = client.ParticipantGroups.add_perticipants_to_group( + id='PPPPPPPPPPPPPPPPPPPPPPPPP', participant_ids=['pppppppppppppppppppppppp1', 'pppppppppppppppppppppppp2'], ) ``` @@ -286,13 +357,15 @@ participant_list: List[Participant] = prolific_api.ParticipantGroups.add_pertici #### Remove Perticipants from Participant Group ```python -from mephisto.abstractions.providers.prolific import api as prolific_api from mephisto.abstractions.providers.prolific.api.data_models import Participant +from mephisto.abstractions.providers.prolific.prolific_utils import get_authenticated_client -participant_list: List[Participant] = prolific_api.ParticipantGroups.remove_perticipants_from_group( - id='PPPPPPPPPPPPPPPPPPPPPPPPP', +client = get_authenticated_client("prolific") +participant_list: List[Participant] = client.ParticipantGroups.remove_perticipants_from_group( + id='PPPPPPPPPPPPPPPPPPPPPPPPP', participant_ids=['pppppppppppppppppppppppp1', 'pppppppppppppppppppppppp2'], ) +``` @@ -305,10 +378,11 @@ look at `mephisto.abstractions.providers.prolific.api.data_models.BonusPayments` #### Set Up Bonus Payments ```python -from mephisto.abstractions.providers.prolific import api as prolific_api from mephisto.abstractions.providers.prolific.api.data_models import BonusPayments +from mephisto.abstractions.providers.prolific.prolific_utils import get_authenticated_client -bonus_payments: BonusPayments = prolific_api.Bonuses.set_up( +client = get_authenticated_client("prolific") +bonus_payments: BonusPayments = client.Bonuses.set_up( study_id='XXXXXXXXXXXXXXXXXXXXXXXXX', csv_bonuses='60ffe5c8371090c7041d43f8,4.25\n60ff44a1d00991f1dfe405d9,4.25', ) @@ -317,14 +391,15 @@ bonus_payments: BonusPayments = prolific_api.Bonuses.set_up( #### Pay ```python -from mephisto.abstractions.providers.prolific import api as prolific_api from mephisto.abstractions.providers.prolific.api.data_models import BonusPayments +from mephisto.abstractions.providers.prolific.prolific_utils import get_authenticated_client -bonus_payments: BonusPayments = prolific_api.Bonuses.set_up( +client = get_authenticated_client("prolific") +bonus_payments: BonusPayments = client.Bonuses.set_up( study_id='XXXXXXXXXXXXXXXXXXXXXXXXX', csv_bonuses='60ffe5c8371090c7041d43f8,4.25\n60ff44a1d00991f1dfe405d9,4.25', ) -prolific_api.Bonuses.pay(id=bonus_payments.id) +client.Bonuses.pay(id=bonus_payments.id) ``` @@ -339,12 +414,13 @@ look at `mephisto.abstractions.providers.prolific.api.data_models.Message` ```python from datetime import datetime, timedelta -from mephisto.abstractions.providers.prolific import api as prolific_api from mephisto.abstractions.providers.prolific.api.data_models import Message +from mephisto.abstractions.providers.prolific.prolific_utils import get_authenticated_client -message_list: List[Message] = prolific_api.Messages.list(user_id='XXXXXXXXXXXXXXXXXXXXXXXXX') +client = get_authenticated_client("prolific") +message_list: List[Message] = client.Messages.list(user_id='XXXXXXXXXXXXXXXXXXXXXXXXX') # Or -message_list: List[Message] = prolific_api.Messages.list( +message_list: List[Message] = client.Messages.list( created_after=(datetime.now() - timedelta(days=10)), ) ``` @@ -352,19 +428,21 @@ message_list: List[Message] = prolific_api.Messages.list( #### List Unread ```python -from mephisto.abstractions.providers.prolific import api as prolific_api from mephisto.abstractions.providers.prolific.api.data_models import Message +from mephisto.abstractions.providers.prolific.prolific_utils import get_authenticated_client -message_list: List[Message] = prolific_api.Messages.list_unread() +client = get_authenticated_client("prolific") +message_list: List[Message] = client.Messages.list_unread() ``` #### Send ```python -from mephisto.abstractions.providers.prolific import api as prolific_api from mephisto.abstractions.providers.prolific.api.data_models import Message +from mephisto.abstractions.providers.prolific.prolific_utils import get_authenticated_client -message: Message = prolific_api.Messages.send( +client = get_authenticated_client("prolific") +message: Message = client.Messages.send( body='Message body', recipient_id='XXXXXXXXXXXXXXXXXXXXXXXXX', study_id='YYYYYYYYYYYYYYYYYYYYYYYYY', @@ -382,12 +460,13 @@ look at `mephisto.abstractions.providers.prolific.api.data_models.Submission` #### List ```python -from mephisto.abstractions.providers.prolific import api as prolific_api from mephisto.abstractions.providers.prolific.api.data_models import Submission +from mephisto.abstractions.providers.prolific.prolific_utils import get_authenticated_client -submission_list: List[Submission] = prolific_api.Submissions.list() +client = get_authenticated_client("prolific") +submission_list: List[Submission] = client.Submissions.list() # or -submission_list: List[Submission] = prolific_api.Submissions.list( +submission_list: List[Submission] = client.Submissions.list( study_id='XXXXXXXXXXXXXXXXXXXXXXXX', ) ``` @@ -395,28 +474,31 @@ submission_list: List[Submission] = prolific_api.Submissions.list( #### Retrieve ```python -from mephisto.abstractions.providers.prolific import api as prolific_api from mephisto.abstractions.providers.prolific.api.data_models import Submission +from mephisto.abstractions.providers.prolific.prolific_utils import get_authenticated_client -submission: Submission = prolific_api.Submissions.retrieve(id='XXXXXXXXXXXXXXXXXXXXXXXX') +client = get_authenticated_client("prolific") +submission: Submission = client.Submissions.retrieve(id='XXXXXXXXXXXXXXXXXXXXXXXX') ``` #### Approve ```python -from mephisto.abstractions.providers.prolific import api as prolific_api from mephisto.abstractions.providers.prolific.api.data_models import Submission +from mephisto.abstractions.providers.prolific.prolific_utils import get_authenticated_client -submission: Submission = prolific_api.Submissions.approve(id='XXXXXXXXXXXXXXXXXXXXXXXX') +client = get_authenticated_client("prolific") +submission: Submission = client.Submissions.approve(id='XXXXXXXXXXXXXXXXXXXXXXXX') ``` #### Reject ```python -from mephisto.abstractions.providers.prolific import api as prolific_api from mephisto.abstractions.providers.prolific.api.data_models import Submission +from mephisto.abstractions.providers.prolific.prolific_utils import get_authenticated_client -submission: Submission = prolific_api.Submissions.reject(id='XXXXXXXXXXXXXXXXXXXXXXXX') +client = get_authenticated_client("prolific") +submission: Submission = client.Submissions.reject(id='XXXXXXXXXXXXXXXXXXXXXXXX') ``` @@ -430,18 +512,20 @@ look at `mephisto.abstractions.providers.prolific.api.data_models.EligibilityReq #### List ```python -from mephisto.abstractions.providers.prolific import api as prolific_api from mephisto.abstractions.providers.prolific.api.data_models import EligibilityRequirement +from mephisto.abstractions.providers.prolific.prolific_utils import get_authenticated_client -submission_list: List[EligibilityRequirement] = prolific_api.EligibilityRequirements.list() +client = get_authenticated_client("prolific") +submission_list: List[EligibilityRequirement] = client.EligibilityRequirements.list() ``` #### Count participants ```python -from mephisto.abstractions.providers.prolific import api as prolific_api +from mephisto.abstractions.providers.prolific.prolific_utils import get_authenticated_client -count: int = prolific_api.EligibilityRequirements.count_participants() +client = get_authenticated_client("prolific") +count: int = client.EligibilityRequirements.count_participants() ``` diff --git a/mephisto/abstractions/providers/prolific/prolific_utils.py b/mephisto/abstractions/providers/prolific/prolific_utils.py index 2b3bf00bf..2af65fc0d 100644 --- a/mephisto/abstractions/providers/prolific/prolific_utils.py +++ b/mephisto/abstractions/providers/prolific/prolific_utils.py @@ -44,6 +44,7 @@ logger = get_logger(name=__name__) +# --- Credentials --- def check_credentials(client: ProlificClient = DEFAULT_CLIENT) -> bool: """Check whether API KEY is correct""" try: @@ -128,7 +129,10 @@ def _convert_eligibility_requirements(value: List[dict]) -> List[dict]: inspect.isclass, ) log_classes_dicts = [ - {"name": c[0], **{p: "" for p in c[1].params()}} for c in available_classes + { + "name": c[0], + **{p: "" for p in c[1].params()} + } for c in available_classes ] logger.info( f"Available Eligibility Requirements in short form for config:\n" @@ -160,6 +164,7 @@ def check_balance(client: ProlificClient, **kwargs) -> Union[float, int, None]: return workspace_balance.available_balance +# --- Workspaces --- def _find_prolific_workspace( client: ProlificClient, title: str, @@ -207,6 +212,7 @@ def find_or_create_prolific_workspace( return workspace +# --- Projects --- def _find_prolific_project( client: ProlificClient, workspace_id: str, @@ -253,6 +259,52 @@ def find_or_create_prolific_project( return project +# --- Eligibility Requirements --- +def _convert_eligibility_requirements(value: List[dict]) -> List[dict]: + """ + Convert short format of Eligibility Requirements to Prolific format + :param value: list of dicts such as + [{ + # Without `web.eligibility.models.` as API requires. We will add it in the code + 'name': 'AgeRangeEligibilityRequirement', + # Prolific Eligibility Requirement attributes and its values + 'min_age': 18, + 'max_age': 100, + }, ...] + :return: reformatted Prolific value, prepared for API request + """ + eligibility_requirements = [] + + try: + for conf_eligibility_requirement in value: + name = conf_eligibility_requirement.get("name") + + if cls := getattr(eligibility_requirement_classes, name, None): + cls_kwargs = {} + for param_name in cls.params(): + if param_name in conf_eligibility_requirement: + cls_kwargs[param_name] = conf_eligibility_requirement[param_name] + eligibility_requirements.append(cls(**cls_kwargs).to_prolific_dict()) + except Exception: + logger.exception("Could not convert passed Eligibility Requirements") + # Generate human-readable log what Eligibility Requirements and with what parameters + # are available. + available_classes = inspect.getmembers( + sys.modules[eligibility_requirement_classes.__name__], inspect.isclass, + ) + log_classes_dicts = [{ + "name": c[0], + **{p: "" for p in c[1].params()} + } for c in available_classes] + logger.info( + f"Available Eligibility Requirements in short form for config:\n" + + "\n".join([str(i) for i in log_classes_dicts]) + ) + raise + + return eligibility_requirements + + def delete_qualification(client: ProlificClient, id: str) -> bool: """ Delete a qualification (Prolific Participant Group) by ID @@ -334,6 +386,7 @@ def find_or_create_qualification( return qualification +# --- Studies --- def _ec2_external_url(task_run_config: "DictConfig") -> str: c = constants url = ec2_architect.get_full_domain(args=task_run_config) @@ -524,6 +577,7 @@ def is_study_expired(study: Study) -> bool: return is_completed and name_with_expired_mark +# --- Workers/Participants --- def add_workers_to_qualification( client: ProlificClient, workers_ids: List[str], @@ -724,6 +778,7 @@ def calculate_pay_amount( return total_cost +# --- Submissions --- def _find_submission( client: ProlificClient, study_id: str,