From c398c6177bf8dacb7657b3474ff9dca2dd9183af Mon Sep 17 00:00:00 2001 From: Glenn Matthews Date: Thu, 30 Jun 2022 16:00:38 -0400 Subject: [PATCH] Version 1.5.1 (#119) * New Example06 - IP Prefixes (#111) * first draft of example06 Co-authored-by: Glenn Matthews * Attempt to fix the read the docs pipeline. (#115) * Attempt to fix the read the docs pipeline. * Yamllint. Co-authored-by: Leo Kirchner * Update CODEOWNERS (#113) * Fix get() by modelname (#118) * Update example05 (#107) * Update example05 * Use site as children * Add update after adding children * Add pylint disable until Redis code is in * Update example * simplify * wip * wip * Update example * Take redis from main * imprort order * yml * update readme * Use diffsync from pypi * Apply suggestions from code review Co-authored-by: Glenn Matthews * Code review * replace bash by python exec * Rename dockerfile to Dockerfile * Update docs source Co-authored-by: Glenn Matthews * Update CHANGELOG and bump version Co-authored-by: Christian Adell Co-authored-by: Leo Kirchner Co-authored-by: Leo Kirchner --- .github/CODEOWNERS | 2 +- .gitignore | 3 + .readthedocs.yml | 7 +- CHANGELOG.md | 15 +++ diffsync/store/__init__.py | 8 +- docs/requirements.txt | 5 + docs/source/api/diffsync.rst | 8 ++ docs/source/api/diffsync.store.local.rst | 7 + docs/source/api/diffsync.store.redis.rst | 7 + docs/source/api/diffsync.store.rst | 14 ++ docs/source/examples/index.rst | 2 + docs/source/template/api/package.rst_t | 16 +-- examples/05-nautobot-peeringdb/Dockerfile | 14 ++ examples/05-nautobot-peeringdb/README.md | 49 ++++--- .../05-nautobot-peeringdb/adapter_nautobot.py | 94 +++++-------- .../adapter_peeringdb.py | 14 +- .../05-nautobot-peeringdb/creds.example.env | 1 + .../05-nautobot-peeringdb/docker-compose.yml | 17 +++ examples/05-nautobot-peeringdb/main.py | 20 ++- examples/05-nautobot-peeringdb/models.py | 4 +- .../05-nautobot-peeringdb/requirements.txt | 3 +- examples/06-ip-prefixes/README.md | 123 ++++++++++++++++++ examples/06-ip-prefixes/adapter_ipam_a.py | 83 ++++++++++++ examples/06-ip-prefixes/adapter_ipam_b.py | 88 +++++++++++++ examples/06-ip-prefixes/data/ipam_a.yml | 16 +++ examples/06-ip-prefixes/data/ipam_b.yml | 11 ++ examples/06-ip-prefixes/main.py | 13 ++ examples/06-ip-prefixes/models.py | 16 +++ examples/06-ip-prefixes/requirements.txt | 2 + pyproject.toml | 2 +- tests/unit/test_diffsync.py | 4 + tests/unit/test_examples.py | 8 ++ 32 files changed, 568 insertions(+), 108 deletions(-) create mode 100644 docs/requirements.txt create mode 100644 docs/source/api/diffsync.store.local.rst create mode 100644 docs/source/api/diffsync.store.redis.rst create mode 100644 docs/source/api/diffsync.store.rst create mode 100644 examples/05-nautobot-peeringdb/Dockerfile create mode 100644 examples/05-nautobot-peeringdb/creds.example.env create mode 100644 examples/05-nautobot-peeringdb/docker-compose.yml create mode 100644 examples/06-ip-prefixes/README.md create mode 100644 examples/06-ip-prefixes/adapter_ipam_a.py create mode 100644 examples/06-ip-prefixes/adapter_ipam_b.py create mode 100644 examples/06-ip-prefixes/data/ipam_a.yml create mode 100644 examples/06-ip-prefixes/data/ipam_b.yml create mode 100755 examples/06-ip-prefixes/main.py create mode 100644 examples/06-ip-prefixes/models.py create mode 100644 examples/06-ip-prefixes/requirements.txt diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 00607c42..636091a7 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,2 +1,2 @@ # Default owners for all files in this repository -* @glennmatthews @dgarros +* @glennmatthews @Kircheneer @chadell diff --git a/.gitignore b/.gitignore index 78344b0b..72211f93 100644 --- a/.gitignore +++ b/.gitignore @@ -289,3 +289,6 @@ fabric.properties ## Sphinx Documentation ## docs/build + +## Secrets +creds.env diff --git a/.readthedocs.yml b/.readthedocs.yml index 8d33d319..cd658436 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -7,9 +7,6 @@ sphinx: fail_on_warning: false python: - version: 3.7 + version: "3.7" install: - - method: "pip" - path: "." - extra_requirements: - - "docs" + - requirements: "docs/requirements.txt" diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a978ab1..5eb09711 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,20 @@ # Changelog +## v1.5.1 - 2022-06-30 + +### Added + +- #111 - Added example 6, regarding IP prefixes. + +### Changed + +- #107 - Updated example 5 to use the Redis backend store. + +### Fixed + +- #115 - Fixed ReadTheDocs rendering pipeline +- #118 - Fixed a regression in `DiffSync.get(modelname, identifiers)` introduced in 1.5.0 + ## v1.5.0 - 2022-06-07 ### Added diff --git a/diffsync/store/__init__.py b/diffsync/store/__init__.py index 9f234afc..d2de62f9 100644 --- a/diffsync/store/__init__.py +++ b/diffsync/store/__init__.py @@ -194,9 +194,9 @@ def _get_object_class_and_model( """Get object class and model name for a model.""" if isinstance(model, str): modelname = model - if not hasattr(self, model): + if not hasattr(self.diffsync, model): return None, modelname - object_class = getattr(self, model) + object_class = getattr(self.diffsync, model) else: object_class = model modelname = model.get_type() @@ -216,7 +216,7 @@ def _get_uid( uid = object_class.create_unique_id(**identifier) else: raise ValueError( - f"Invalid args: ({model}, {identifier}): " - f"either {model} should be a class/instance or {identifier} should be a str" + f"Invalid args: ({model}, {object_class}, {identifier}): " + f"either {object_class} should be a class/instance or {identifier} should be a str" ) return uid diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 00000000..5f7db800 --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,5 @@ +m2r2==0.2.7 +mistune==0.8.4 +sphinx==4.5.0 +toml==0.10.2 +sphinx-rtd-theme==1.0.0 \ No newline at end of file diff --git a/docs/source/api/diffsync.rst b/docs/source/api/diffsync.rst index 36349fc7..f0829933 100644 --- a/docs/source/api/diffsync.rst +++ b/docs/source/api/diffsync.rst @@ -16,3 +16,11 @@ API Reference diffsync.helpers diffsync.logging diffsync.utils + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + diffsync.store diff --git a/docs/source/api/diffsync.store.local.rst b/docs/source/api/diffsync.store.local.rst new file mode 100644 index 00000000..e6f342c6 --- /dev/null +++ b/docs/source/api/diffsync.store.local.rst @@ -0,0 +1,7 @@ +diffsync.store.local +==================== + +.. automodule:: diffsync.store.local + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/api/diffsync.store.redis.rst b/docs/source/api/diffsync.store.redis.rst new file mode 100644 index 00000000..64576153 --- /dev/null +++ b/docs/source/api/diffsync.store.redis.rst @@ -0,0 +1,7 @@ +diffsync.store.redis +==================== + +.. automodule:: diffsync.store.redis + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/api/diffsync.store.rst b/docs/source/api/diffsync.store.rst new file mode 100644 index 00000000..35638d29 --- /dev/null +++ b/docs/source/api/diffsync.store.rst @@ -0,0 +1,14 @@ +API Reference +============= + +.. automodule:: diffsync.store + :members: + :undoc-members: + :show-inheritance: + + +.. toctree:: + :maxdepth: 4 + + diffsync.store.local + diffsync.store.redis diff --git a/docs/source/examples/index.rst b/docs/source/examples/index.rst index 4bd1043c..d67609c7 100644 --- a/docs/source/examples/index.rst +++ b/docs/source/examples/index.rst @@ -8,3 +8,5 @@ For each example, the complete source code is `available in Github The source code for this example is in Github in the [examples/05-nautobot-peeringdb/](https://github.com/networktocode/diffsync/tree/main/examples/05-nautobot-peeringdb) directory. -## Install dependencies +## Get PeeringDB API Key (optional) + +To ensure a good performance from PeeringDB API, you should provide an API Key: https://docs.peeringdb.com/howto/api_keys/ + +Then, copy the example `creds.example.env` into `creds.env`, and place your new API Key. ```bash -python3 -m venv .venv -source .venv/bin/activate -pip3 install -r requirements.txt +$ cp examples/05-nautobot-peeringdb/creds.example.env examples/05-nautobot-peeringdb/creds.env + ``` -## Run it interactively +> Without API Key it might also work, but it could fail due to API rate limiting. -```python -from IPython import embed -embed(colors="neutral") +## Set up local docker environment -# Import Adapters -from diffsync.enum import DiffSyncFlags +```bash +$ docker-compose -f examples/05-nautobot-peeringdb/docker-compose.yml up -d --build + +$ docker exec -it 05-nautobot-peeringdb_example_1 python +``` + +## Interactive execution +```python from adapter_nautobot import NautobotRemote from adapter_peeringdb import PeeringDB +from diffsync.enum import DiffSyncFlags +from diffsync.store.redis import RedisStore + +store_one = RedisStore(host="redis") +store_two = RedisStore(host="redis") # Initialize PeeringDB adapter, using CATNIX id for demonstration -peeringdb = PeeringDB(ix_id=62) +peeringdb = PeeringDB( + ix_id=62, + internal_storage_engine=store_one +) # Initialize Nautobot adapter, pointing to the demo instance (it's also the default settings) nautobot = NautobotRemote( url="https://demo.nautobot.com", - token="aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + token="a" * 40, + internal_storage_engine=store_two ) # Load PeeringDB info into the adapter @@ -55,12 +71,11 @@ peeringdb.dict() nautobot.load() # Let's diffsync do it's magic -diff = nautobot.diff_from(peeringdb) +diff = nautobot.diff_from(peeringdb, flags=DiffSyncFlags.SKIP_UNMATCHED_DST) # Quick summary of the expected changes (remember that delete ones are dry-run) diff.summary() # Execute the synchronization nautobot.sync_from(peeringdb, flags=DiffSyncFlags.SKIP_UNMATCHED_DST) - ``` diff --git a/examples/05-nautobot-peeringdb/adapter_nautobot.py b/examples/05-nautobot-peeringdb/adapter_nautobot.py index e86597e0..7970274d 100644 --- a/examples/05-nautobot-peeringdb/adapter_nautobot.py +++ b/examples/05-nautobot-peeringdb/adapter_nautobot.py @@ -1,15 +1,10 @@ """Diffsync adapter class for Nautobot.""" # pylint: disable=import-error,no-name-in-module -import os -import requests +import pynautobot from models import RegionModel, SiteModel from diffsync import DiffSync -NAUTOBOT_URL = os.getenv("NAUTOBOT_URL", "https://demo.nautobot.com") -NAUTOBOT_TOKEN = os.getenv("NAUTOBOT_TOKEN", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") - - class RegionNautobotModel(RegionModel): """Implementation of Region create/update/delete methods for updating remote Nautobot data.""" @@ -30,7 +25,9 @@ def create(cls, diffsync, ids, attrs): data["description"] = attrs["description"] if attrs["parent_name"]: data["parent"] = str(diffsync.get(diffsync.region, attrs["parent_name"]).pk) - diffsync.post("/api/dcim/regions/", data) + + diffsync.nautobot_api.dcim.regions.create(**data) + return super().create(diffsync, ids=ids, attrs=attrs) def update(self, attrs): @@ -39,6 +36,7 @@ def update(self, attrs): Args: attrs (dict): Updated values for this record's _attributes """ + region = self.diffsync.nautobot_api.dcim.regions.get(name=self.name) data = {} if "slug" in attrs: data["slug"] = attrs["slug"] @@ -46,15 +44,17 @@ def update(self, attrs): data["description"] = attrs["description"] if "parent_name" in attrs: if attrs["parent_name"]: - data["parent"] = str(self.diffsync.get(self.diffsync.region, attrs["parent_name"]).pk) + data["parent"] = str(self.diffsync.get(self.diffsync.region, attrs["parent_name"]).name) else: data["parent"] = None - self.diffsync.patch(f"/api/dcim/regions/{self.pk}/", data) + + region.update(data=data) + return super().update(attrs) def delete(self): # pylint: disable= useless-super-delegation """Delete an existing Region record from remote Nautobot.""" - # self.diffsync.delete(f"/api/dcim/regions/{self.pk}/") + # Not implemented return super().delete() @@ -70,17 +70,14 @@ def create(cls, diffsync, ids, attrs): ids (dict): Initial values for this model's _identifiers attrs (dict): Initial values for this model's _attributes """ - diffsync.post( - "/api/dcim/sites/", - { - "name": ids["name"], - "slug": attrs["slug"], - "description": attrs["description"], - "status": attrs["status_slug"], - "region": {"name": attrs["region_name"]} if attrs["region_name"] else None, - "latitude": attrs["latitude"], - "longitude": attrs["longitude"], - }, + diffsync.nautobot_api.dcim.sites.create( + name=ids["name"], + slug=attrs["slug"], + description=attrs["description"], + status=attrs["status_slug"], + region={"name": attrs["region_name"]} if attrs["region_name"] else None, + latitude=attrs["latitude"], + longitude=attrs["longitude"], ) return super().create(diffsync, ids=ids, attrs=attrs) @@ -90,6 +87,8 @@ def update(self, attrs): Args: attrs (dict): Updated values for this record's _attributes """ + site = self.diffsync.nautobot_api.dcim.sites.get(name=self.name) + data = {} if "slug" in attrs: data["slug"] = attrs["slug"] @@ -106,12 +105,14 @@ def update(self, attrs): data["latitude"] = attrs["latitude"] if "longitude" in attrs: data["longitude"] = attrs["longitude"] - self.diffsync.patch(f"/api/dcim/sites/{self.pk}/", data) + + site.update(data=data) + return super().update(attrs) def delete(self): # pylint: disable= useless-super-delegation """Delete an existing Site record from remote Nautobot.""" - # self.diffsync.delete(f"/api/dcim/sites/{self.pk}/") + # Not implemented return super().delete() @@ -123,9 +124,9 @@ class NautobotRemote(DiffSync): site = SiteNautobotModel # Top-level class labels, i.e. those classes that are handled directly rather than as children of other models - top_level = ("region", "site") + top_level = ["region"] - def __init__(self, *args, url=NAUTOBOT_URL, token=NAUTOBOT_TOKEN, **kwargs): + def __init__(self, *args, url, token, **kwargs): """Instantiate this class, but do not load data immediately from the remote system. Args: @@ -136,21 +137,11 @@ def __init__(self, *args, url=NAUTOBOT_URL, token=NAUTOBOT_TOKEN, **kwargs): super().__init__(*args, **kwargs) if not url or not token: raise ValueError("Both url and token must be specified!") - self.url = url - self.token = token - self.headers = { - "Accept": "application/json", - "Authorization": f"Token {self.token}", - } + self.nautobot_api = pynautobot.api(url=url, token=token) def load(self): """Load Region and Site data from the remote Nautobot instance.""" - region_data = requests.get(f"{self.url}/api/dcim/regions/", headers=self.headers, params={"limit": 0}).json() - regions = region_data["results"] - while region_data["next"]: - region_data = requests.get(region_data["next"], headers=self.headers, params={"limit": 0}).json() - regions.extend(region_data["results"]) - + regions = self.nautobot_api.dcim.regions.all() for region_entry in regions: region = self.region( name=region_entry["name"], @@ -161,12 +152,7 @@ def load(self): ) self.add(region) - site_data = requests.get(f"{self.url}/api/dcim/sites/", headers=self.headers, params={"limit": 0}).json() - sites = site_data["results"] - while site_data["next"]: - site_data = requests.get(site_data["next"], headers=self.headers, params={"limit": 0}).json() - sites.extend(site_data["results"]) - + sites = self.nautobot_api.dcim.sites.all() for site_entry in sites: site = self.site( name=site_entry["name"], @@ -179,21 +165,7 @@ def load(self): pk=site_entry["id"], ) self.add(site) - - def post(self, path, data): - """Send an appropriately constructed HTTP POST request.""" - response = requests.post(f"{self.url}{path}", headers=self.headers, json=data) - response.raise_for_status() - return response - - def patch(self, path, data): - """Send an appropriately constructed HTTP PATCH request.""" - response = requests.patch(f"{self.url}{path}", headers=self.headers, json=data) - response.raise_for_status() - return response - - def delete(self, path): - """Send an appropriately constructed HTTP DELETE request.""" - response = requests.delete(f"{self.url}{path}", headers=self.headers) - response.raise_for_status() - return response + if site_entry["region"]: + region = self.get(self.region, site_entry["region"]["name"]) + region.add_child(site) + self.update(region) diff --git a/examples/05-nautobot-peeringdb/adapter_peeringdb.py b/examples/05-nautobot-peeringdb/adapter_peeringdb.py index 0bd6616e..08feb600 100644 --- a/examples/05-nautobot-peeringdb/adapter_peeringdb.py +++ b/examples/05-nautobot-peeringdb/adapter_peeringdb.py @@ -1,5 +1,6 @@ """Diffsync adapter class for PeeringDB.""" # pylint: disable=import-error,no-name-in-module +import os import requests from slugify import slugify import pycountry @@ -9,6 +10,7 @@ PEERINGDB_URL = "https://peeringdb.com/" +PEERINGDB_API_KEY = os.environ.get("PEERINGDB_API_KEY", "").strip() class PeeringDB(DiffSync): @@ -19,7 +21,7 @@ class PeeringDB(DiffSync): site = SiteModel # Top-level class labels, i.e. those classes that are handled directly rather than as children of other models - top_level = ("region", "site") + top_level = ["region"] def __init__(self, *args, ix_id, **kwargs): """Initialize the PeeringDB adapter.""" @@ -28,12 +30,16 @@ def __init__(self, *args, ix_id, **kwargs): def load(self): """Load data via from PeeringDB.""" - ix_data = requests.get(f"{PEERINGDB_URL}/api/ix/{self.ix_id}").json() + headers = {} + if PEERINGDB_API_KEY: + headers["Authorization"] = f"Api-Key {PEERINGDB_API_KEY}" + + ix_data = requests.get(f"{PEERINGDB_URL}/api/ix/{self.ix_id}", headers=headers).json() for fac in ix_data["data"][0]["fac_set"]: # PeeringDB has no Region entity, so we must avoid duplicates try: - self.get(self.region, fac["city"]) + region = self.get(self.region, fac["city"]) except ObjectNotFound: # Use pycountry to translate the country code (like "DE") to a country name (like "Germany") parent_name = pycountry.countries.get(alpha_2=fac["country"]).name @@ -65,3 +71,5 @@ def load(self): pk=fac["id"], ) self.add(site) + region.add_child(site) + self.update(region) # pylint: disable=no-member diff --git a/examples/05-nautobot-peeringdb/creds.example.env b/examples/05-nautobot-peeringdb/creds.example.env new file mode 100644 index 00000000..5e8ff7f4 --- /dev/null +++ b/examples/05-nautobot-peeringdb/creds.example.env @@ -0,0 +1 @@ +PEERINGDB_API_KEY="" diff --git a/examples/05-nautobot-peeringdb/docker-compose.yml b/examples/05-nautobot-peeringdb/docker-compose.yml new file mode 100644 index 00000000..1315d3d7 --- /dev/null +++ b/examples/05-nautobot-peeringdb/docker-compose.yml @@ -0,0 +1,17 @@ +--- +version: "3.8" +services: + example: + build: + context: "./" + dockerfile: "Dockerfile" + tty: true + depends_on: + redis: + condition: "service_started" + volumes: + - "./:/local" + env_file: + - "creds.env" + redis: + image: "redis:6-alpine" diff --git a/examples/05-nautobot-peeringdb/main.py b/examples/05-nautobot-peeringdb/main.py index c2cb7943..6f011830 100644 --- a/examples/05-nautobot-peeringdb/main.py +++ b/examples/05-nautobot-peeringdb/main.py @@ -5,13 +5,21 @@ from adapter_peeringdb import PeeringDB from diffsync.enum import DiffSyncFlags +from diffsync.store.redis import RedisStore +REDIS_HOST = "redis" +PEERING_DB_IX_ID = 62 # CATNIX ID +NAUTOBOT_URL = "https://demo.nautobot.com" +NAUTOBOT_TOKEN = "a" * 40 -# Initialize PeeringDB adapter, using CATNIX id for demonstration -peeringdb = PeeringDB(ix_id=62) +store_one = RedisStore(host=REDIS_HOST) +store_two = RedisStore(host=REDIS_HOST) + +# Initialize PeeringDB adapter +peeringdb = PeeringDB(ix_id=PEERING_DB_IX_ID, internal_storage_engine=store_one) # Initialize Nautobot adapter, pointing to the demo instance (it's also the default settings) -nautobot = NautobotRemote(url="https://demo.nautobot.com", token="a" * 40) # nosec +nautobot = NautobotRemote(url=NAUTOBOT_URL, token=NAUTOBOT_TOKEN, internal_storage_engine=store_two) # nosec # Load PeeringDB info into the adapter peeringdb.load() @@ -19,13 +27,13 @@ # We can check the data that has been imported, some as `site` and some as `region` (with the parent relationships) peeringdb.dict() -# Load Nautobot info into the adapter +# Load Nautobot info into the Nautobot adapter nautobot.load() # Let's diffsync do it's magic -diff = nautobot.diff_from(peeringdb) +diff = nautobot.diff_from(peeringdb, flags=DiffSyncFlags.SKIP_UNMATCHED_DST) -# Quick summary of the expected changes (remember that delete ones are dry-run) +# Quick summary of the expected changes diff.summary() # Execute the synchronization diff --git a/examples/05-nautobot-peeringdb/models.py b/examples/05-nautobot-peeringdb/models.py index 063ef1f6..2fecb044 100644 --- a/examples/05-nautobot-peeringdb/models.py +++ b/examples/05-nautobot-peeringdb/models.py @@ -1,5 +1,5 @@ """DiffSyncModel subclasses for Nautobot-PeeringDB data sync.""" -from typing import Optional, Union +from typing import Optional, Union, List from uuid import UUID from diffsync import DiffSyncModel @@ -16,12 +16,14 @@ class RegionModel(DiffSyncModel): "description", "parent_name", ) + _children = {"site": "sites"} # Data type declarations for all identifiers and attributes name: str slug: str description: Optional[str] parent_name: Optional[str] # may be None + sites: List = [] # Not in _attributes or _identifiers, hence not included in diff calculations pk: Optional[UUID] diff --git a/examples/05-nautobot-peeringdb/requirements.txt b/examples/05-nautobot-peeringdb/requirements.txt index 84af760c..dba2709e 100644 --- a/examples/05-nautobot-peeringdb/requirements.txt +++ b/examples/05-nautobot-peeringdb/requirements.txt @@ -1,5 +1,6 @@ -diffsync +diffsync[redis]>=1.5.0 python-slugify pycountry requests IPython +pynautobot diff --git a/examples/06-ip-prefixes/README.md b/examples/06-ip-prefixes/README.md new file mode 100644 index 00000000..e1e7266b --- /dev/null +++ b/examples/06-ip-prefixes/README.md @@ -0,0 +1,123 @@ +# Example 6 - IP Prefixes + +This example shows how to play around to IPAM systems which have a different implementation of an IP Prefix. + +These IPAM systems, IPAM A and IPAM B, are simulated using two YAML files within the `data` folder. These files are dynamic, and they will be loaded and updated from diffsync. + +## Test the example + +You could simply run the `main.py` file, but to run step by step. + +### Set up the environment + +Install the dependencies (recommended into a virtual environment) + +``` +pip3 install -r requirements.txt +``` + +and go into a `python` interactive session: + +``` +python3 +>>> +``` + +### Import the DiffSync adapters + +```py +>>> from adapter_ipam_a import IpamA +>>> from adapter_ipam_b import IpamB +``` + +### Initialize and load adapter for IPAM A + +```py +>>> ipam_a = IpamA() +>>> ipam_a.load() +``` + +You can check the content loaded from IPAM A. Notice that the data has been transformed into the DiffSync model, which is different from the original YAML data. + +```py +>>> import pprint +>>> pprint.pprint(ipam_a.dict()) +{'prefix': {'10.10.10.10/24': {'prefix': '10.10.10.10/24', + 'vlan_id': 10, + 'vrf': 'data'}, + '10.20.20.20/24': {'prefix': '10.20.20.20/24', + 'tenant': 'ABC corp', + 'vlan_id': 20, + 'vrf': 'voice'}, + '172.18.0.0/16': {'prefix': '172.18.0.0/16', 'vlan_id': 18}}} +``` + +### Initialize and load adapter for IPAM B + +```py +>>> ipam_b = IpamB() +>>> ipam_b.load() +``` + +You can check the content loaded from IPAM B. Notice that the data has been transformed into the DiffSync model, which again is different from the original YAML format. + +```py +>>> pprint.pprint(ipam_b.dict()) +{'prefix': {'10.10.10.10/24': {'prefix': '10.10.10.10/24', 'vlan_id': 123}, + '2001:DB8::/32': {'prefix': '2001:DB8::/32', + 'tenant': 'XYZ Corporation', + 'vlan_id': 10, + 'vrf': 'data'}}} +``` + +### Check the difference + +We can use `diff_to` or `diff_from` to select, from the perspective of the calling adapter, who is the authoritative in each case. + +```py +>>> diff = ipam_a.diff_to(ipam_b) +``` + +From this `diff`, we can check the summary of what would happen. + +```py +>>> diff.summary() +{'create': 2, 'update': 1, 'delete': 1, 'no-change': 0} +``` + +And, also go into the details. We can see how the `'+'` and + `'-'` represent the actual changes in the target adapter: create, delete or update (when both symbols appear). + +```py +>>> pprint.pprint(diff.dict()) +{'prefix': {'10.10.10.10/24': {'+': {'vlan_id': 10, 'vrf': 'data'}, + '-': {'vlan_id': 123, 'vrf': None}}, + '10.20.20.20/24': {'+': {'tenant': 'ABC corp', + 'vlan_id': 20, + 'vrf': 'voice'}}, + '172.18.0.0/16': {'+': {'tenant': None, + 'vlan_id': 18, + 'vrf': None}}, + '2001:DB8::/32': {'-': {'tenant': 'XYZ Corporation', + 'vlan_id': 10, + 'vrf': 'data'}}}} +``` + +### Enforce synchronization + +Simply transforming the `diff_to` to `sync_to`, we are going to change the state of the destination target. + +```py +>>> ipam_a.sync_to(ipam_b) +``` + +### Validate synchronization + +Now, if we reload the IPAM B, and try to check the difference, we should see no differences. + +```py +>>> new_ipam_b = IpamB() +>>> new_ipam_b.load() +>>> diff = ipam_a.diff_to(new_ipam_b) +>>> diff.summary() +{'create': 0, 'update': 0, 'delete': 0, 'no-change': 3} +``` diff --git a/examples/06-ip-prefixes/adapter_ipam_a.py b/examples/06-ip-prefixes/adapter_ipam_a.py new file mode 100644 index 00000000..4b43595e --- /dev/null +++ b/examples/06-ip-prefixes/adapter_ipam_a.py @@ -0,0 +1,83 @@ +"""IPAM A adapter.""" +import os +import ipaddress +import yaml +from models import Prefix # pylint: disable=no-name-in-module +from diffsync import DiffSync + +dirname = os.path.dirname(os.path.realpath(__file__)) + + +class IpamAPrefix(Prefix): + """Implementation of Prefix create/update/delete methods for IPAM A.""" + + @classmethod + def create(cls, diffsync, ids, attrs): + """Create a Prefix record in IPAM A.""" + diffsync.data.append( + { + "cidr": ids["prefix"], + "family": ipaddress.ip_address(ids["prefix"].split("/")[0]).version, + "vrf": attrs["vrf"], + "vlan": f'VLAN{attrs["vlan_id"]}', + "customer_id": attrs["tenant"] if attrs["tenant"] else None, + } + ) + + return super().create(diffsync, ids=ids, attrs=attrs) + + def update(self, attrs): + """Update a Prefix record in IPAM A.""" + for elem in self.diffsync.data: + if elem["cidr"] == self.prefix: + if "vrf" in attrs: + elem["vrf"] = attrs["vrf"] + if "vlan_id" in attrs: + elem["vlan_id"] = f'VLAN{attrs["vlan_id"]}' + if "tenant" in attrs: + elem["customer_id"] = attrs["tenant"] + break + + return super().update(attrs) + + def delete(self): + """Delete a Prefix record in IPAM A.""" + for index, elem in enumerate(self.diffsync.data): + if elem["cidr"] == self.prefix: + del self.diffsync.data[index] + break + + return super().delete() + + +class IpamA(DiffSync): + """IPAM A DiffSync adapter implementation.""" + + prefix = IpamAPrefix + + top_level = ["prefix"] + + def __init__(self, *args, **kwargs): + """Initialize the IPAM A Adapter.""" + super().__init__(*args, **kwargs) + + with open(os.path.join(dirname, "data", "ipam_a.yml"), encoding="utf-8") as data_file: + self.data = yaml.safe_load(data_file) + + def load(self): + """Load prefixes from IPAM A.""" + for subnet in self.data: + prefix = self.prefix( + prefix=subnet["cidr"], + vrf=subnet["vrf"], + vlan_id=int(subnet["vlan"].lstrip("VLAN")), + tenant=subnet["customer_id"], + ) + self.add(prefix) + + def sync_complete(self, source, *args, **kwargs): + """Clean up function for DiffSync sync.""" + with open(os.path.join(dirname, "data", "ipam_a.yml"), encoding="utf-8", mode="w") as data_file: + yaml.safe_dump(self.data, data_file) + + return super().sync_complete(source, *args, **kwargs) diff --git a/examples/06-ip-prefixes/adapter_ipam_b.py b/examples/06-ip-prefixes/adapter_ipam_b.py new file mode 100644 index 00000000..b29e76b8 --- /dev/null +++ b/examples/06-ip-prefixes/adapter_ipam_b.py @@ -0,0 +1,88 @@ +"""IPAM B adapter.""" +import os +import yaml +from models import Prefix # pylint: disable=no-name-in-module +from diffsync import DiffSync + +dirname = os.path.dirname(os.path.realpath(__file__)) + + +class IpamBPrefix(Prefix): + """Implementation of Prefix create/update/delete methods for IPAM B.""" + + @classmethod + def create(cls, diffsync, ids, attrs): + """Create a Prefix record in IPAM B.""" + diffsync.data.append( + { + "network": ids["prefix"].split("/")[0], + "prefix_length": int(ids["prefix"].split("/")[1]), + "vrf": attrs["vrf"], + "vlan_id": attrs["vlan_id"], + "tenant": attrs["tenant"] if attrs["tenant"] else None, + } + ) + + return super().create(diffsync, ids=ids, attrs=attrs) + + def update(self, attrs): + """Update a Prefix record in IPAM B.""" + network = self.prefix.split("/")[0] + prefix_length = int(self.prefix.split("/")[1]) + + for elem in self.diffsync.data: + if elem["network"] == network and elem["prefix_length"] == prefix_length: + if "vrf" in attrs: + elem["vrf"] = attrs["vrf"] + if "vlan_id" in attrs: + elem["vlan_id"] = attrs["vlan_id"] + if "tenant" in attrs: + elem["tenant"] = attrs["tenant"] + break + + return super().update(attrs) + + def delete(self): + """Update a Prefix record in IPAM B.""" + network = self.prefix.split("/")[0] + prefix_length = int(self.prefix.split("/")[1]) + + for index, elem in enumerate(self.diffsync.data): + if elem["network"] == network and elem["prefix_length"] == prefix_length: + del self.diffsync.data[index] + break + + return super().delete() + + +class IpamB(DiffSync): + """IPAM A DiffSync adapter implementation.""" + + prefix = IpamBPrefix + + top_level = ["prefix"] + + def __init__(self, *args, **kwargs): + """Initialize the IPAM B Adapter.""" + super().__init__(*args, **kwargs) + + with open(os.path.join(dirname, "data", "ipam_b.yml"), encoding="utf-8") as data_file: + self.data = yaml.safe_load(data_file) + + def load(self): + """Initialize the Ipam B Object by loading from DATA.""" + for prefix_data in self.data: + prefix = self.prefix( + prefix=f"{prefix_data['network']}/{prefix_data['prefix_length']}", + vrf=prefix_data["vrf"], + vlan_id=prefix_data["vlan_id"], + tenant=prefix_data["tenant"], + ) + self.add(prefix) + + def sync_complete(self, source, *args, **kwargs): + """Clean up function for DiffSync sync.""" + with open(os.path.join(dirname, "data", "ipam_b.yml"), encoding="utf-8", mode="w") as data_file: + yaml.safe_dump(self.data, data_file) + + return super().sync_complete(source, *args, **kwargs) diff --git a/examples/06-ip-prefixes/data/ipam_a.yml b/examples/06-ip-prefixes/data/ipam_a.yml new file mode 100644 index 00000000..ca6603fc --- /dev/null +++ b/examples/06-ip-prefixes/data/ipam_a.yml @@ -0,0 +1,16 @@ +--- +- cidr: "10.10.10.10/24" + family: 4 + vrf: "data" + vlan: "VLAN10" + customer_id: null +- cidr: "10.20.20.20/24" + family: 4 + vrf: "voice" + vlan: "VLAN20" + customer_id: "ABC corp" +- cidr: "172.18.0.0/16" + family: 4 + vrf: null + vlan: "VLAN18" + customer_id: null diff --git a/examples/06-ip-prefixes/data/ipam_b.yml b/examples/06-ip-prefixes/data/ipam_b.yml new file mode 100644 index 00000000..1cf68b49 --- /dev/null +++ b/examples/06-ip-prefixes/data/ipam_b.yml @@ -0,0 +1,11 @@ +--- +- network: "10.10.10.10" + prefix_length: 24 + tenant: null + vlan_id: 123 + vrf: "voice" +- network: "2001:DB8::" + prefix_length: 32 + tenant: "XYZ Corporation" + vlan_id: 10 + vrf: "data" diff --git a/examples/06-ip-prefixes/main.py b/examples/06-ip-prefixes/main.py new file mode 100755 index 00000000..26088141 --- /dev/null +++ b/examples/06-ip-prefixes/main.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python +"""Main example.""" +from adapter_ipam_a import IpamA +from adapter_ipam_b import IpamB + + +if __name__ == "__main__": + ipam_a = IpamA() + ipam_b = IpamB() + ipam_a.load() + ipam_b.load() + diff = ipam_a.diff_to(ipam_b) + # ipam_a.sync_to(ipam_b) diff --git a/examples/06-ip-prefixes/models.py b/examples/06-ip-prefixes/models.py new file mode 100644 index 00000000..2fc4abdd --- /dev/null +++ b/examples/06-ip-prefixes/models.py @@ -0,0 +1,16 @@ +"""DiffSync models.""" +from typing import Optional +from diffsync import DiffSyncModel + + +class Prefix(DiffSyncModel): + """Example model of a Prefix.""" + + _modelname = "prefix" + _identifiers = ("prefix",) + _attributes = ("vrf", "vlan_id", "tenant") + + prefix: str + vrf: Optional[str] + vlan_id: Optional[int] + tenant: Optional[str] diff --git a/examples/06-ip-prefixes/requirements.txt b/examples/06-ip-prefixes/requirements.txt new file mode 100644 index 00000000..d1e22a7a --- /dev/null +++ b/examples/06-ip-prefixes/requirements.txt @@ -0,0 +1,2 @@ +diffsync +pyyaml diff --git a/pyproject.toml b/pyproject.toml index ada51b50..1f472807 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "diffsync" -version = "1.5.0" +version = "1.5.1" description = "Library to easily sync/diff/update 2 different data sources" authors = ["Network to Code, LLC "] license = "Apache-2.0" diff --git a/tests/unit/test_diffsync.py b/tests/unit/test_diffsync.py index 9e8ccb33..c9e1f26b 100644 --- a/tests/unit/test_diffsync.py +++ b/tests/unit/test_diffsync.py @@ -115,6 +115,7 @@ def test_diffsync_add_raises_already_exists_with_updated_object(generic_diffsync def test_diffsync_get_or_instantiate_create_non_existent_object(generic_diffsync): + generic_diffsync.interface = Interface intf_identifiers = {"device_name": "device1", "name": "eth1"} # Assert that the object does not currently exist. @@ -124,6 +125,7 @@ def test_diffsync_get_or_instantiate_create_non_existent_object(generic_diffsync obj, created = generic_diffsync.get_or_instantiate(Interface, intf_identifiers) assert created assert obj is generic_diffsync.get(Interface, intf_identifiers) + assert obj is generic_diffsync.get("interface", intf_identifiers) def test_diffsync_get_or_instantiate_retrieve_existing_object(generic_diffsync): @@ -150,6 +152,7 @@ def test_diffsync_get_or_instantiate_retrieve_existing_object_w_attrs(generic_di def test_diffsync_get_or_instantiate_retrieve_create_non_existent_w_attrs(generic_diffsync): + generic_diffsync.interface = Interface intf_identifiers = {"device_name": "device1", "name": "eth1"} intf_attrs = {"interface_type": "1000base-t", "description": "Testing"} @@ -158,6 +161,7 @@ def test_diffsync_get_or_instantiate_retrieve_create_non_existent_w_attrs(generi assert obj.interface_type == "1000base-t" assert obj.description == "Testing" assert obj is generic_diffsync.get(Interface, intf_identifiers) + assert obj is generic_diffsync.get("interface", intf_identifiers) def test_diffsync_get_or_instantiate_retrieve_existing_object_wo_attrs(generic_diffsync): diff --git a/tests/unit/test_examples.py b/tests/unit/test_examples.py index 41119488..2f9af164 100644 --- a/tests/unit/test_examples.py +++ b/tests/unit/test_examples.py @@ -43,3 +43,11 @@ def test_example_4(): example4_main = join(example4_dir, "main.py") # Run it and make sure it doesn't raise an exception or otherwise exit with a non-zero code. subprocess.run(example4_main, cwd=example4_dir, check=True) + + +def test_example_6(): + """Test that the "example6" script runs successfully.""" + example6_dir = join(EXAMPLES, "06-ip-prefixes") + example6_main = join(example6_dir, "main.py") + # Run it and make sure it doesn't raise an exception or otherwise exit with a non-zero code. + subprocess.run(example6_main, cwd=example6_dir, check=True)