diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 947fcf5c..0f8dfeca 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,7 +16,6 @@ repos: rev: v3.1.0 hooks: - id: add-trailing-comma - args: [--py36-plus] - repo: https://github.com/asottile/pyupgrade rev: v3.3.1 hooks: diff --git a/README.md b/README.md index 6a2cf135..a5637083 100644 --- a/README.md +++ b/README.md @@ -714,6 +714,40 @@ def test_pebble_exec(): ) ``` +### Pebble Notices + +Pebble can generate notices, which Juju will detect, and wake up the charm to +let it know that something has happened in the container. The most common +use-case is Pebble custom notices, which is a mechanism for the workload +application to trigger a charm event. + +When the charm is notified, there might be a queue of existing notices, or just +the one that has triggered the event: + +```python +import ops +import scenario + +class MyCharm(ops.CharmBase): + def __init__(self, framework): + super().__init__(framework) + framework.observe(self.on["cont"].pebble_custom_notice, self._on_notice) + + def _on_notice(self, event): + event.notice.key # == "example.com/c" + for notice in self.unit.get_container("cont").get_notices(): + ... + +ctx = scenario.Context(MyCharm, meta={"name": "foo", "containers": {"my-container": {}}}) +notices = [ + scenario.Notice(key="example.com/a", occurences=10), + scenario.Notice(key="example.com/b", last_data={"bar": "baz"}), + scenario.Notice(key="example.com/c"), +] +cont = scenario.Container(notices=notices) +ctx.run(container.get_notice("example.com/c").event, scenario.State(containers=[cont])) +``` + ## Storage If your charm defines `storage` in its metadata, you can use `scenario.Storage` to instruct Scenario to make (mocked) filesystem storage available to the charm at runtime. diff --git a/pyproject.toml b/pyproject.toml index fcf9a915..5907ae71 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ build-backend = "setuptools.build_meta" [project] name = "ops-scenario" -version = "6.0.5" +version = "6.1.0" authors = [ { name = "Pietro Pasotti", email = "pietro.pasotti@canonical.com" } @@ -18,7 +18,7 @@ license.text = "Apache-2.0" keywords = ["juju", "test"] dependencies = [ - "ops>=2.6", + "ops>=2.10", "PyYAML>=6.0.1", ] readme = "README.md" diff --git a/scenario/__init__.py b/scenario/__init__.py index 8b832ab0..dfe567f0 100644 --- a/scenario/__init__.py +++ b/scenario/__init__.py @@ -15,6 +15,7 @@ Model, Mount, Network, + Notice, PeerRelation, Port, Relation, @@ -45,6 +46,7 @@ "ExecOutput", "Mount", "Container", + "Notice", "Address", "BindAddress", "Network", diff --git a/scenario/consistency_checker.py b/scenario/consistency_checker.py index 584bad67..e73602a4 100644 --- a/scenario/consistency_checker.py +++ b/scenario/consistency_checker.py @@ -532,14 +532,15 @@ def check_containers_consistency( meta = charm_spec.meta meta_containers = list(map(normalize_name, meta.get("containers", {}))) state_containers = [normalize_name(c.name) for c in state.containers] + all_notices = {notice.id for c in state.containers for notice in c.notices} errors = [] # it's fine if you have containers in meta that are not in state.containers (yet), but it's # not fine if: - # - you're processing a pebble-ready event and that container is not in state.containers or + # - you're processing a Pebble event and that container is not in state.containers or # meta.containers if event._is_workload_event: - evt_container_name = event.name[: -len("-pebble-ready")] + evt_container_name = event.name.split("_pebble_")[0] if evt_container_name not in meta_containers: errors.append( f"the event being processed concerns container {evt_container_name!r}, but a " @@ -548,8 +549,13 @@ def check_containers_consistency( if evt_container_name not in state_containers: errors.append( f"the event being processed concerns container {evt_container_name!r}, but a " - f"container with that name is not present in the state. It's odd, but consistent, " - f"if it cannot connect; but it should at least be there.", + f"container with that name is not present in the state. It's odd, but " + f"consistent, if it cannot connect; but it should at least be there.", + ) + if event.notice and event.notice.id not in all_notices: + errors.append( + f"the event being processed concerns notice {event.notice!r}, but that " + "notice is not in any of the containers present in the state.", ) # - a container in state.containers is not in meta.containers diff --git a/scenario/mocking.py b/scenario/mocking.py index a55397d2..af72a361 100644 --- a/scenario/mocking.py +++ b/scenario/mocking.py @@ -675,6 +675,16 @@ def __init__( self._root = container_root + # load any existing notices from the state + self._notices: Dict[Tuple[str, str], pebble.Notice] = {} + for container in state.containers: + for notice in container.notices: + if hasattr(notice.type, "value"): + notice_type = cast(pebble.NoticeType, notice.type).value + else: + notice_type = str(notice.type) + self._notices[notice_type, notice.key] = notice._to_ops() + def get_plan(self) -> pebble.Plan: return self._container.plan diff --git a/scenario/runtime.py b/scenario/runtime.py index a6aebab0..3cb67f0b 100644 --- a/scenario/runtime.py +++ b/scenario/runtime.py @@ -11,6 +11,7 @@ from typing import TYPE_CHECKING, Dict, List, Optional, Type, Union import yaml +from ops import pebble from ops.framework import _event_regex from ops.storage import NoSnapshotError, SQLiteStorage @@ -248,6 +249,19 @@ def _get_event_env(self, state: "State", event: "Event", charm_root: Path): if container := event.container: env.update({"JUJU_WORKLOAD_NAME": container.name}) + if notice := event.notice: + if hasattr(notice.type, "value"): + notice_type = typing.cast(pebble.NoticeType, notice.type).value + else: + notice_type = str(notice.type) + env.update( + { + "JUJU_NOTICE_ID": notice.id, + "JUJU_NOTICE_TYPE": notice_type, + "JUJU_NOTICE_KEY": notice.key, + }, + ) + if storage := event.storage: env.update({"JUJU_STORAGE_ID": f"{storage.name}/{storage.index}"}) diff --git a/scenario/state.py b/scenario/state.py index c0d573fa..3fb0a26a 100644 --- a/scenario/state.py +++ b/scenario/state.py @@ -88,6 +88,7 @@ "collect_unit_status", } PEBBLE_READY_EVENT_SUFFIX = "_pebble_ready" +PEBBLE_CUSTOM_NOTICE_EVENT_SUFFIX = "_pebble_custom_notice" RELATION_EVENTS_SUFFIX = { "_relation_changed", "_relation_broken", @@ -683,6 +684,96 @@ class Mount(_DCBase): src: Union[str, Path] +def _now_utc(): + return datetime.datetime.now(tz=datetime.timezone.utc) + + +_next_notice_id_counter = 1 + + +def next_notice_id(update=True): + global _next_notice_id_counter + cur = _next_notice_id_counter + if update: + _next_notice_id_counter += 1 + return str(cur) + + +@dataclasses.dataclass(frozen=True) +class Notice(_DCBase): + key: str + """The notice key, a string that differentiates notices of this type. + + This is in the format ``domain/path``; for example: + ``canonical.com/postgresql/backup`` or ``example.com/mycharm/notice``. + """ + + id: str = dataclasses.field(default_factory=next_notice_id) + """Unique ID for this notice.""" + + user_id: Optional[int] = None + """UID of the user who may view this notice (None means notice is public).""" + + type: Union[pebble.NoticeType, str] = pebble.NoticeType.CUSTOM + """Type of the notice.""" + + first_occurred: datetime.datetime = dataclasses.field(default_factory=_now_utc) + """The first time one of these notices (type and key combination) occurs.""" + + last_occurred: datetime.datetime = dataclasses.field(default_factory=_now_utc) + """The last time one of these notices occurred.""" + + last_repeated: datetime.datetime = dataclasses.field(default_factory=_now_utc) + """The time this notice was last repeated. + + See Pebble's `Notices documentation `_ + for an explanation of what "repeated" means. + """ + + occurrences: int = 1 + """The number of times one of these notices has occurred.""" + + last_data: Dict[str, str] = dataclasses.field(default_factory=dict) + """Additional data captured from the last occurrence of one of these notices.""" + + repeat_after: Optional[datetime.timedelta] = None + """Minimum time after one of these was last repeated before Pebble will repeat it again.""" + + expire_after: Optional[datetime.timedelta] = None + """How long since one of these last occurred until Pebble will drop the notice.""" + + def _to_ops(self) -> pebble.Notice: + return pebble.Notice( + id=self.id, + user_id=self.user_id, + type=self.type, + key=self.key, + first_occurred=self.first_occurred, + last_occurred=self.last_occurred, + last_repeated=self.last_repeated, + occurrences=self.occurrences, + last_data=self.last_data, + repeat_after=self.repeat_after, + expire_after=self.expire_after, + ) + + +@dataclasses.dataclass(frozen=True) +class _BoundNotice(_DCBase): + notice: Notice + container: "Container" + + @property + def event(self): + """Sugar to generate a -pebble-custom-notice event for this notice.""" + suffix = PEBBLE_CUSTOM_NOTICE_EVENT_SUFFIX + return Event( + path=normalize_name(self.container.name) + suffix, + container=self.container, + notice=self.notice, + ) + + @dataclasses.dataclass(frozen=True) class Container(_DCBase): name: str @@ -720,6 +811,8 @@ class Container(_DCBase): exec_mock: _ExecMock = dataclasses.field(default_factory=dict) + notices: List[Notice] = dataclasses.field(default_factory=list) + def _render_services(self): # copied over from ops.testing._TestingPebbleClient._render_services() services = {} # type: Dict[str, pebble.Service] @@ -787,6 +880,23 @@ def pebble_ready_event(self): ) return Event(path=normalize_name(self.name + "-pebble-ready"), container=self) + def get_notice( + self, + key: str, + notice_type: pebble.NoticeType = pebble.NoticeType.CUSTOM, + ) -> _BoundNotice: + """Get a Pebble notice by key and type. + + Raises: + KeyError: if the notice is not found. + """ + for notice in self.notices: + if notice.key == key and notice.type == notice_type: + return _BoundNotice(notice, self) + raise KeyError( + f"{self.name} does not have a notice with key {key} and type {notice_type}", + ) + _RawStatusLiteral = Literal[ "waiting", @@ -1265,6 +1375,8 @@ def _get_suffix_and_type(s: str) -> Tuple[str, _EventType]: # Whether the event name indicates that this is a workload event. if s.endswith(PEBBLE_READY_EVENT_SUFFIX): return PEBBLE_READY_EVENT_SUFFIX, _EventType.workload + if s.endswith(PEBBLE_CUSTOM_NOTICE_EVENT_SUFFIX): + return PEBBLE_CUSTOM_NOTICE_EVENT_SUFFIX, _EventType.workload if s in BUILTIN_EVENTS: return "", _EventType.builtin @@ -1291,6 +1403,9 @@ class Event(_DCBase): # if this is a workload (container) event, the container it refers to container: Optional[Container] = None + # if this is a Pebble notice event, the notice it refers to + notice: Optional[Notice] = None + # if this is an action event, the Action instance action: Optional["Action"] = None @@ -1471,6 +1586,18 @@ def deferred(self, handler: Callable, event_id: int = 1) -> DeferredEvent: snapshot_data = { "container_name": container.name, } + if self.notice: + if hasattr(self.notice.type, "value"): + notice_type = cast(pebble.NoticeType, self.notice.type).value + else: + notice_type = str(self.notice.type) + snapshot_data.update( + { + "notice_id": self.notice.id, + "notice_key": self.notice.key, + "notice_type": notice_type, + }, + ) elif self._is_relation_event: # this is a RelationEvent. @@ -1534,8 +1661,9 @@ def deferred( event_id: int = 1, relation: Optional["Relation"] = None, container: Optional["Container"] = None, + notice: Optional["Notice"] = None, ): """Construct a DeferredEvent from an Event or an event name.""" if isinstance(event, str): - event = Event(event, relation=relation, container=container) + event = Event(event, relation=relation, container=container, notice=notice) return event.deferred(handler=handler, event_id=event_id) diff --git a/tests/test_consistency_checker.py b/tests/test_consistency_checker.py index 78c85102..707cc7f4 100644 --- a/tests/test_consistency_checker.py +++ b/tests/test_consistency_checker.py @@ -12,6 +12,7 @@ Container, Event, Network, + Notice, PeerRelation, Relation, Secret, @@ -63,6 +64,22 @@ def test_workload_event_without_container(): Event("foo-pebble-ready", container=Container("foo")), _CharmSpec(MyCharm, {"containers": {"foo": {}}}), ) + assert_inconsistent( + State(), + Event("foo-pebble-custom-notice", container=Container("foo")), + _CharmSpec(MyCharm, {}), + ) + notice = Notice("example.com/foo") + assert_consistent( + State(containers=[Container("foo", notices=[notice])]), + Event("foo-pebble-custom-notice", container=Container("foo"), notice=notice), + _CharmSpec(MyCharm, {"containers": {"foo": {}}}), + ) + assert_inconsistent( + State(containers=[Container("foo")]), + Event("foo-pebble-custom-notice", container=Container("foo"), notice=notice), + _CharmSpec(MyCharm, {"containers": {"foo": {}}}), + ) def test_container_meta_mismatch(): diff --git a/tests/test_e2e/test_deferred.py b/tests/test_e2e/test_deferred.py index b084f6ff..a96100bc 100644 --- a/tests/test_e2e/test_deferred.py +++ b/tests/test_e2e/test_deferred.py @@ -12,7 +12,7 @@ from ops.framework import Framework from scenario import Context -from scenario.state import Container, DeferredEvent, Relation, State, deferred +from scenario.state import Container, DeferredEvent, Notice, Relation, State, deferred from tests.helpers import trigger CHARM_CALLED = 0 @@ -97,6 +97,20 @@ def test_deferred_workload_evt(mycharm): assert asdict(evt2) == asdict(evt1) +def test_deferred_notice_evt(mycharm): + notice = Notice(key="example.com/bar") + ctr = Container("foo", notices=[notice]) + evt1 = ctr.get_notice("example.com/bar").event.deferred(handler=mycharm._on_event) + evt2 = deferred( + event="foo_pebble_custom_notice", + handler=mycharm._on_event, + container=ctr, + notice=notice, + ) + + assert asdict(evt2) == asdict(evt1) + + def test_deferred_relation_event(mycharm): mycharm.defer_next = 2 diff --git a/tests/test_e2e/test_event.py b/tests/test_e2e/test_event.py index 0dd50077..07c8d30a 100644 --- a/tests/test_e2e/test_event.py +++ b/tests/test_e2e/test_event.py @@ -17,6 +17,8 @@ ("foo_bar_baz_storage_detaching", _EventType.storage), ("foo_pebble_ready", _EventType.workload), ("foo_bar_baz_pebble_ready", _EventType.workload), + ("foo_pebble_custom_notice", _EventType.workload), + ("foo_bar_baz_pebble_custom_notice", _EventType.workload), ("secret_removed", _EventType.secret), ("pre_commit", _EventType.framework), ("commit", _EventType.framework), diff --git a/tests/test_e2e/test_pebble.py b/tests/test_e2e/test_pebble.py index b87b350d..e5c16bf7 100644 --- a/tests/test_e2e/test_pebble.py +++ b/tests/test_e2e/test_pebble.py @@ -1,14 +1,15 @@ +import datetime import tempfile from pathlib import Path import pytest -from ops import pebble +from ops import PebbleCustomNoticeEvent, pebble from ops.charm import CharmBase from ops.framework import Framework from ops.pebble import ExecError, ServiceStartup, ServiceStatus from scenario import Context -from scenario.state import Container, ExecOutput, Mount, Port, State +from scenario.state import Container, ExecOutput, Mount, Notice, Port, State from tests.helpers import trigger @@ -365,3 +366,76 @@ def test_exec_wait_output_error(charm_cls): proc = container.exec(["foo"]) with pytest.raises(ExecError): proc.wait_output() + + +def test_pebble_custom_notice(charm_cls): + notices = [ + Notice(key="example.com/foo"), + Notice(key="example.com/bar", last_data={"a": "b"}), + Notice(key="example.com/baz", occurrences=42), + ] + container = Container( + name="foo", + can_connect=True, + notices=notices, + ) + + state = State(containers=[container]) + ctx = Context(charm_cls, meta={"name": "foo", "containers": {"foo": {}}}) + with ctx.manager(container.get_notice("example.com/baz").event, state) as mgr: + container = mgr.charm.unit.get_container("foo") + assert container.get_notices() == [n._to_ops() for n in notices] + + +def test_pebble_custom_notice_in_charm(): + key = "example.com/test/charm" + data = {"foo": "bar"} + user_id = 100 + first_occurred = datetime.datetime(1979, 1, 25, 11, 0, 0) + last_occured = datetime.datetime(2006, 8, 28, 13, 28, 0) + last_repeated = datetime.datetime(2023, 9, 4, 9, 0, 0) + occurrences = 42 + repeat_after = datetime.timedelta(days=7) + expire_after = datetime.timedelta(days=365) + + class MyCharm(CharmBase): + def __init__(self, framework): + super().__init__(framework) + framework.observe(self.on.foo_pebble_custom_notice, self._on_custom_notice) + + def _on_custom_notice(self, event: PebbleCustomNoticeEvent): + notice = event.notice + assert notice.type == pebble.NoticeType.CUSTOM + assert notice.key == key + assert notice.last_data == data + assert notice.user_id == user_id + assert notice.first_occurred == first_occurred + assert notice.last_occurred == last_occured + assert notice.last_repeated == last_repeated + assert notice.occurrences == occurrences + assert notice.repeat_after == repeat_after + assert notice.expire_after == expire_after + + notices = [ + Notice("example.com/test/other"), + Notice("example.org/test/charm", last_data={"foo": "baz"}), + Notice( + key, + last_data=data, + user_id=user_id, + first_occurred=first_occurred, + last_occurred=last_occured, + last_repeated=last_repeated, + occurrences=occurrences, + repeat_after=repeat_after, + expire_after=expire_after, + ), + ] + container = Container( + name="foo", + can_connect=True, + notices=notices, + ) + state = State(containers=[container]) + ctx = Context(MyCharm, meta={"name": "foo", "containers": {"foo": {}}}) + ctx.run(container.get_notice(key).event, state) diff --git a/tests/test_e2e/test_state.py b/tests/test_e2e/test_state.py index d92e5b12..838426ae 100644 --- a/tests/test_e2e/test_state.py +++ b/tests/test_e2e/test_state.py @@ -19,6 +19,7 @@ "storage_detaching", "action", "pebble_ready", + "pebble_custom_notice", }