Skip to content

Commit

Permalink
Merge pull request #1100 from facebookresearch/v1.2.1-dev
Browse files Browse the repository at this point in the history
  • Loading branch information
meta-paul authored Feb 1, 2024
2 parents aac76d7 + e56b5c0 commit 5686c28
Show file tree
Hide file tree
Showing 19 changed files with 266 additions and 87 deletions.
69 changes: 68 additions & 1 deletion examples/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,71 @@
-->

# Examples
The Mephisto example folders within contain some sample starter code and tasks to demonstrate potential workflows for setting up and working on new tasks.
The Mephisto example folders within contain some sample starter code and tasks to demonstrate potential workflows for setting up and working on new tasks.

Mephisto Tasks can be launched (each run is called TaskRun) with a single `docker-compose` command (you will need to have Docker [installed](https://docs.docker.com/engine/install/).)

Let's launch Mephisto example tasks, starting from the easiest one

---

#### 1. Simple HTML-based task

A simple project with HTML-based UI task template [simple_static_task](/examples/simple_static_task)

- Default config file: [/examples/simple_static_task/hydra_configs/conf/example.yaml]
- Launch command:
```shell
docker-compose -f docker/docker-compose.dev.yml run \
--build \
--publish 3001:3000 \
--rm mephisto_dc \
python /mephisto/examples/simple_static_task/static_test_script.py
```
- Browser page (for the first task unit): [http://localhost:3001/?worker_id=x&assignment_id=1](http://localhost:3001/?worker_id=x&assignment_id=1)
- Browser page should display an image, instruction, select and file inputs, and a submit button.

---

#### 2. Simple React-based task

A simple project with React-based UI task template [static_react_task](/examples/static_react_task)

- Default config file: [example.yaml](/examples/static_react_task/hydra_configs/conf/example.yaml).
- Launch command:
```shell
docker-compose -f docker/docker-compose.dev.yml run \
--build \
--publish 3001:3000 \
--rm mephisto_dc \
python /mephisto/examples/static_react_task/run_task.py
```
- Browser page (for the first task unit): [http://localhost:3001/?worker_id=x&assignment_id=1](http://localhost:3001/?worker_id=x&assignment_id=1).
- Browser page should display an instruction line and two buttons (green and red).

---

#### 3. Task with dynamic input

A more complex example featuring worker-generated dynamic input: [mnist](/examples/remote_procedure/mnist).

- Default config file: [launch_with_local.yaml](/examples/remote_procedure/mnist/hydra_configs/conf/launch_with_local.yaml).
- Launch command:
```shell
docker-compose -f docker/docker-compose.dev.yml run \
--build \
--publish 3001:3000 \
--rm mephisto_dc \
apt install curl && \
pip install grafana torch pillow numpy && \
mephisto metrics install && \
python /mephisto/examples/remote_procedure/mnist/run_task.py
```
- Browser page (for the first task unit): [http://localhost:3001/?worker_id=x&assignment_id=1](http://localhost:3001/?worker_id=x&assignment_id=1).
- Browser page should display instructions and a layout with 3 rectangle fields for drawing numbers with a mouse, each field having inputs at the bottom.

---

# Your Mephisto project

To read on steps for creating your own custom Mephisto task, please refer to README in the main Mephisto repo.
14 changes: 9 additions & 5 deletions mephisto/abstractions/providers/mock/mock_agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,17 +4,21 @@
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

from mephisto.data_model.agent import Agent
from typing import Any
from typing import Dict
from typing import Mapping
from typing import Optional
from typing import TYPE_CHECKING
from typing import Union

from mephisto.abstractions.blueprint import AgentState
from mephisto.abstractions.providers.mock.provider_type import PROVIDER_TYPE

from typing import List, Optional, Tuple, Dict, Mapping, Any, TYPE_CHECKING
from mephisto.data_model.agent import Agent

if TYPE_CHECKING:
from mephisto.data_model.unit import Unit
from mephisto.abstractions.database import MephistoDB
from mephisto.data_model.worker import Worker
from mephisto.data_model.packet import Packet
from mephisto.abstractions.providers.mock.mock_datastore import MockDatastore


Expand Down Expand Up @@ -76,7 +80,7 @@ def get_live_update(self, timeout=None) -> Optional[Dict[str, Any]]:
def approve_work(
self,
review_note: Optional[str] = None,
bonus: Optional[str] = None,
bonus: Optional[Union[int, float]] = None,
skip_unit_review: bool = False,
) -> None:
"""
Expand Down
22 changes: 18 additions & 4 deletions mephisto/abstractions/providers/mock/mock_worker.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,22 +4,30 @@
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

from mephisto.data_model.worker import Worker
from typing import Any
from typing import Mapping
from typing import Optional
from typing import Tuple
from typing import TYPE_CHECKING

from mephisto.abstractions.providers.mock.provider_type import PROVIDER_TYPE
from typing import List, Optional, Tuple, Dict, Mapping, Type, Any, TYPE_CHECKING
from mephisto.data_model.worker import Worker
from mephisto.utils.logger_core import get_logger

if TYPE_CHECKING:
from mephisto.abstractions.database import MephistoDB
from mephisto.data_model.task_run import TaskRun
from mephisto.data_model.unit import Unit
from mephisto.data_model.agent import Agent
from mephisto.data_model.requester import Requester
from mephisto.abstractions.providers.mock.mock_datastore import MockDatastore

logger = get_logger(name=__name__)


class MockWorker(Worker):
"""
This class represents an individual - namely a person. It maintains components of ongoing identity for a user.
This class represents an individual - namely a person.
It maintains components of ongoing identity for a user.
"""

def __init__(
Expand All @@ -36,6 +44,7 @@ def bonus_worker(
self, amount: float, reason: str, unit: Optional["Unit"] = None
) -> Tuple[bool, str]:
"""Bonus this worker for work any reason. Return success of bonus"""
logger.debug(f"Mock paying bonus to worker. Amount: {amount}. Reason: '{reason}")
return True, ""

def block_worker(
Expand All @@ -61,6 +70,11 @@ def is_eligible(self, task_run: "TaskRun") -> bool:
"""Determine if this worker is eligible for the given task run"""
return True

def send_feedback_message(self, text: str, unit: "Unit") -> bool:
"""Send feedback message to a worker"""
logger.debug(f"Mock sending feedback message to worker: '{text}'. Unit: {unit}")
return True

@staticmethod
def new(db: "MephistoDB", worker_id: str) -> "Worker":
return MockWorker._register_worker(db, worker_id + "_sandbox", PROVIDER_TYPE)
32 changes: 17 additions & 15 deletions mephisto/abstractions/providers/mturk/mturk_agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,25 +4,25 @@
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

from mephisto.data_model.agent import Agent
from mephisto.abstractions.blueprint import AgentState
from mephisto.abstractions.providers.mturk.provider_type import PROVIDER_TYPE
from mephisto.abstractions.providers.mturk.mturk_utils import (
approve_work,
reject_work,
get_assignment,
get_assignments_for_hit,
)

import xmltodict # type: ignore
import json
from typing import Any
from typing import cast
from typing import Dict
from typing import Mapping
from typing import Optional
from typing import TYPE_CHECKING
from typing import Union

from typing import List, Optional, Tuple, Dict, Mapping, Any, cast, TYPE_CHECKING
import xmltodict # type: ignore

from mephisto.abstractions.blueprint import AgentState
from mephisto.abstractions.providers.mturk.mturk_utils import approve_work
from mephisto.abstractions.providers.mturk.mturk_utils import get_assignments_for_hit
from mephisto.abstractions.providers.mturk.mturk_utils import reject_work
from mephisto.abstractions.providers.mturk.provider_type import PROVIDER_TYPE
from mephisto.data_model.agent import Agent
from mephisto.utils.logger_core import get_logger

logger = get_logger(name=__name__)

if TYPE_CHECKING:
from mephisto.data_model.unit import Unit
from mephisto.abstractions.database import MephistoDB
Expand All @@ -31,6 +31,8 @@
from mephisto.abstractions.providers.mturk.mturk_unit import MTurkUnit
from mephisto.abstractions.providers.mturk.mturk_datastore import MTurkDatastore

logger = get_logger(name=__name__)


class MTurkAgent(Agent):
"""
Expand Down Expand Up @@ -104,7 +106,7 @@ def attempt_to_reconcile_submitted_data(self, mturk_hit_id: str):
def approve_work(
self,
review_note: Optional[str] = None,
bonus: Optional[str] = None,
bonus: Optional[Union[int, float]] = None,
skip_unit_review: bool = False,
) -> None:
"""Approve the work done on this specific Unit"""
Expand Down
54 changes: 38 additions & 16 deletions mephisto/abstractions/providers/mturk/mturk_worker.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,32 +4,33 @@
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

from mephisto.data_model.worker import Worker
from mephisto.data_model.requester import Requester
from mephisto.abstractions.providers.mturk.provider_type import PROVIDER_TYPE
from mephisto.abstractions.providers.mturk.mturk_utils import (
pay_bonus,
block_worker,
unblock_worker,
is_worker_blocked,
give_worker_qualification,
remove_worker_qualification,
)
from mephisto.abstractions.providers.mturk.mturk_requester import MTurkRequester

from typing import Any
from typing import cast
from typing import Mapping
from typing import Optional
from typing import Tuple
from typing import TYPE_CHECKING
from uuid import uuid4

from typing import List, Optional, Tuple, Dict, Mapping, Any, cast, TYPE_CHECKING
from mephisto.abstractions.providers.mturk.mturk_utils import block_worker
from mephisto.abstractions.providers.mturk.mturk_utils import email_worker
from mephisto.abstractions.providers.mturk.mturk_utils import give_worker_qualification
from mephisto.abstractions.providers.mturk.mturk_utils import is_worker_blocked
from mephisto.abstractions.providers.mturk.mturk_utils import pay_bonus
from mephisto.abstractions.providers.mturk.mturk_utils import remove_worker_qualification
from mephisto.abstractions.providers.mturk.mturk_utils import unblock_worker
from mephisto.abstractions.providers.mturk.provider_type import PROVIDER_TYPE
from mephisto.data_model.requester import Requester
from mephisto.data_model.worker import Worker
from mephisto.utils.logger_core import get_logger

if TYPE_CHECKING:
from mephisto.abstractions.providers.mturk.mturk_datastore import MTurkDatastore
from mephisto.abstractions.database import MephistoDB
from mephisto.data_model.task_run import TaskRun
from mephisto.data_model.unit import Unit
from mephisto.abstractions.providers.mturk.mturk_unit import MTurkUnit
from mephisto.abstractions.providers.mturk.mturk_requester import MTurkRequester

from mephisto.utils.logger_core import get_logger

logger = get_logger(name=__name__)

Expand Down Expand Up @@ -179,6 +180,27 @@ def is_eligible(self, task_run: "TaskRun") -> bool:
"""
return True

def send_feedback_message(self, text: str, unit: "Unit") -> bool:
"""Send feedback message to a worker"""
requester = cast(
"MTurkRequester",
self.db.find_requesters(provider_type=self.provider_type)[-1],
)

assert isinstance(requester, MTurkRequester), "Must be an MTurk requester"

client = self._get_client(requester._requester_name)
task_name = unit.get_task_run().get_task().task_name

email_worker(
client=client,
worker_id=self.get_mturk_worker_id(),
subject=f'Feedback for your Mturk task "{task_name}"',
message_text=text,
)

return True

@staticmethod
def new(db: "MephistoDB", worker_id: str) -> "Worker":
return MTurkWorker._register_worker(db, worker_id, PROVIDER_TYPE)
8 changes: 6 additions & 2 deletions mephisto/abstractions/providers/prolific/api/messages.py
Original file line number Diff line number Diff line change
Expand Up @@ -52,9 +52,13 @@ def list_unread(cls) -> List[Message]:
return messages

@classmethod
def send(cls, **data) -> Message:
def send(cls, study_id: str, recipient_id: str, text: str) -> Message:
"""Send a message to a participant or another researcher"""
message = Message(**data)
message = Message(
body=text,
recipient_id=recipient_id,
study_id=study_id,
)
message.validate()
response_json = cls.post(cls.list_api_endpoint, params=message.to_dict())
return Message(**response_json)
9 changes: 5 additions & 4 deletions mephisto/abstractions/providers/prolific/prolific_agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,14 +10,15 @@
from typing import Mapping
from typing import Optional
from typing import TYPE_CHECKING
from typing import Union

from mephisto.abstractions.blueprint import AgentState
from mephisto.abstractions.providers.prolific import prolific_utils
from mephisto.abstractions.providers.prolific.api.client import ProlificClient
from mephisto.abstractions.providers.prolific.api.constants import SubmissionStatus
from mephisto.abstractions.providers.prolific.provider_type import PROVIDER_TYPE
from mephisto.data_model.agent import Agent
from mephisto.utils.logger_core import get_logger
from mephisto.abstractions.providers.prolific.api.client import ProlificClient
from mephisto.abstractions.providers.prolific.api.constants import SubmissionStatus

if TYPE_CHECKING:
from mephisto.abstractions.providers.prolific.prolific_datastore import ProlificDatastore
Expand Down Expand Up @@ -103,7 +104,7 @@ def new_from_provider_data(
def approve_work(
self,
review_note: Optional[str] = None,
bonus: Optional[str] = None,
bonus: Optional[Union[int, float]] = None,
skip_unit_review: bool = False,
) -> None:
"""Approve the work done on this specific Unit"""
Expand Down Expand Up @@ -139,7 +140,7 @@ def approve_work(
worker_id=unit.worker_id,
status=AgentState.STATUS_APPROVED,
review_note=review_note,
bonus=bonus,
bonus=str(bonus),
)

def soft_reject_work(self, review_note: Optional[str] = None) -> None:
Expand Down
12 changes: 11 additions & 1 deletion mephisto/abstractions/providers/prolific/prolific_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
from .api.base_api_resource import CREDENTIALS_CONFIG_PATH
from .api.client import ProlificClient
from .api.data_models import BonusPayments
from .api.data_models import Message
from .api.data_models import Participant
from .api.data_models import ParticipantGroup
from .api.data_models import Project
Expand Down Expand Up @@ -596,7 +597,7 @@ def pay_bonus(
client: ProlificClient,
task_run_config: "DictConfig",
worker_id: str,
bonus_amount: int, # in cents
bonus_amount: Union[int, float], # in cents
study_id: str,
*args,
**kwargs,
Expand Down Expand Up @@ -798,3 +799,12 @@ def reject_work(
)

return None


def send_message(client: ProlificClient, study_id: str, participant_id: str, text: str) -> Message:
try:
message: Message = client.Messages.send(study_id, participant_id, text)
except (ProlificException, ValidationError):
logger.exception(f'Could not send message to participant "{participant_id}"')
raise
return message
Loading

0 comments on commit 5686c28

Please sign in to comment.