From 4039c6ba1e19d2ccc4d65c48803719984b9a3bfe Mon Sep 17 00:00:00 2001 From: Adam Byczkowski <38091261+qduk@users.noreply.github.com> Date: Mon, 29 Nov 2021 14:35:32 -0600 Subject: [PATCH] Update "group by" process in inventory_graphql (#110) --- docs/plugins/gql_inventory_inventory.rst | 10 +- plugins/inventory/gql_inventory.py | 148 +++++++++--------- .../test_data/graphql_groups/device_data.json | 25 +++ tests/unit/inventory/test_graphql.py | 144 +++++++++++++++++ 4 files changed, 253 insertions(+), 74 deletions(-) create mode 100644 tests/unit/inventory/test_data/graphql_groups/device_data.json create mode 100644 tests/unit/inventory/test_graphql.py diff --git a/docs/plugins/gql_inventory_inventory.rst b/docs/plugins/gql_inventory_inventory.rst index 3b4a51d5..dd9bc92b 100755 --- a/docs/plugins/gql_inventory_inventory.rst +++ b/docs/plugins/gql_inventory_inventory.rst @@ -172,7 +172,7 @@ Parameters -
List of group names to group the hosts
+
List of data paths to group the hosts.
@@ -289,7 +289,7 @@ Examples .. code-block:: yaml+jinja - + # inventory.yml file in YAML format # Example command line: ansible-inventory -v --list -i inventory.yml @@ -309,12 +309,14 @@ Examples - platform # To group by use group_by key - # Please see choices for supported group_by options. + # Specify the full path to the data you would like to use to group by. + # Note. If you pass in a single string rather than a path, the plugin will automatically try to find a name or slug value. plugin: networktocode.nautobot.gql_inventory api_endpoint: http://localhost:8000 validate_certs: True group_by: - - platform + - tenant.name + - status.slug # Add additional variables diff --git a/plugins/inventory/gql_inventory.py b/plugins/inventory/gql_inventory.py index 146b36db..4746072a 100644 --- a/plugins/inventory/gql_inventory.py +++ b/plugins/inventory/gql_inventory.py @@ -2,6 +2,7 @@ # GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) from __future__ import absolute_import, division, print_function +from collections.abc import Mapping __metaclass__ = type @@ -63,10 +64,12 @@ group_by: required: False description: - - List of group names to group the hosts + - List of dot-sparated paths to index graphql query results (e.g. `platform.slug`) + - The final value returned by each path is used to derive group names and then group the devices into these groups. + - Valid group names must be string, so indexing the dotted path should return a string (i.e. `platform.slug` instead of `platform`) + - If value returned by the defined path is a dictionary, an attempt will first be made to access the `name` field, and then the `slug` field. (i.e. `platform` would attempt to lookup `platform.name`, and if that data was not returned, it would then try `platform.slug`) type: list default: [] - choices: ["platform", "status", "device_role", "site"] filters: required: false description: @@ -87,21 +90,13 @@ tags: name # To group by use group_by key -# Please see choices for supported group_by options. +# Specify the full path to the data you would like to use to group by. plugin: networktocode.nautobot.gql_inventory api_endpoint: http://localhost:8000 validate_certs: True group_by: - - platform - -# To group by use group_by key -# Please see choices for supported group_by options. -plugin: networktocode.nautobot.gql_inventory -api_endpoint: http://localhost:8000 -validate_certs: True -group_by: - - platform - + - tenant.name + - status.slug # Add additional variables plugin: networktocode.nautobot.gql_inventory @@ -169,16 +164,6 @@ def verify_file(self, path): return False - def create_inventory(self, group: str, host: str): - """Creates Ansible inventory. - - Args: - group (str): Name of the group - host (str): Hostname - """ - self.inventory.add_group(group) - self.inventory.add_host(host, group) - def add_variable(self, host: str, var: str, var_type: str): """Adds variables to group or host. @@ -189,6 +174,69 @@ def add_variable(self, host: str, var: str, var_type: str): """ self.inventory.set_variable(host, var_type, var) + def add_ipv4_address(self, device): + """Add primary IPv4 address to host.""" + if device["primary_ip4"]: + self.add_variable(device["name"], device["primary_ip4"]["address"], "ansible_host") + else: + self.add_variable(device["name"], device["name"], "ansible_host") + + def add_ansible_platform(self, device): + """Add network platform to host""" + if device["platform"] and "napalm_driver" in device["platform"]: + self.add_variable( + device["name"], + ANSIBLE_LIB_MAPPER_REVERSE.get(NAPALM_LIB_MAPPER.get(device["platform"]["napalm_driver"])), # Convert napalm_driver to ansible_network_os value + "ansible_network_os", + ) + + def populate_variables(self, device): + """Add specified variables to device.""" + for var in self.variables: + if var in device and device[var]: + self.add_variable(device["name"], device[var], var) + + def create_groups(self, device): + """Create groups specified and add device to group.""" + device_name = device["name"] + for group_by_path in self.group_by: + parent_attr, *chain = group_by_path.split(".") + device_attr = device.get(parent_attr) + if device_attr is None: + self.display.display(f"Could not find value for {parent_attr} on device {device_name}") + continue + + if not chain: + group_name = device_attr + + while chain: + group_name = chain.pop(0) + if isinstance(device_attr.get(group_name), Mapping): + device_attr = device_attr.get(group_name) + continue + else: + try: + group_name = device_attr[group_name] + except KeyError: + self.display.display(f"Could not find value for {group_name} in {group_by_path} on device {device_name}.") + break + + if isinstance(group_name, Mapping): + if "name" in group_name: + group_name = group_name["name"] + elif "slug" in group_name: + group_name = group_name["slug"] + else: + self.display.display(f"No slug or name value for {group_name} in {group_by_path} on device {device_name}.") + + if isinstance(group_name, str): + self.inventory.add_group(group_name) + self.inventory.add_child(group_name, device_name) + else: + self.display.display( + f"Groups must be a string. {group_name} is not a string. Please make sure your group_by path specified resolves to a string value." + ) + def main(self): """Main function.""" if not HAS_NETUTILS: @@ -243,53 +291,12 @@ def main(self): # Need to return mock response data that is empty to prevent any failures downstream return {"results": [], "next": None} - groups = {} - if self.group_by: - for group_by in self.group_by: - if not GROUP_BY.get(group_by): - self.display.display( - "WARNING: '{0}' is not supported as a 'group_by' option. Supported options are: {1} ".format( - group_by, - " ".join("'{0}',".format(str(x)) for x in GROUP_BY.keys()), - ), - color="yellow", - ) - continue - for device in json_data["data"]["devices"]: - groups[device["site"]["name"]] = "site" - if device.get(group_by) and GROUP_BY.get(group_by): - groups[device[group_by][GROUP_BY.get(group_by)]] = group_by - else: - groups["unknown"] = "unknown" - - else: - for device in json_data["data"]["devices"]: - groups[device["site"]["name"]] = "site" - - for key, value in groups.items(): - for device in json_data["data"]["devices"]: - if device.get(value) and key == device[value][GROUP_BY[value]]: - self.create_inventory(key, device["name"]) - if device["primary_ip4"]: - self.add_variable( - device["name"], - device["primary_ip4"]["address"], - "ansible_host", - ) - else: - self.add_variable(device["name"], device["name"], "ansible_host") - if device["platform"] and "napalm_driver" in device["platform"]: - self.add_variable( - device["name"], - ANSIBLE_LIB_MAPPER_REVERSE.get( - NAPALM_LIB_MAPPER.get(device["platform"]["napalm_driver"]) # Convert napalm_driver to ansible_network_os value - ), - "ansible_network_os", - ) - - for var in self.variables: - if var in device and device[var]: - self.add_variable(device["name"], device[var], var) + for device in json_data["data"]["devices"]: + self.inventory.add_host(device["name"]) + self.add_ipv4_address(device) + self.add_ansible_platform(device) + self.populate_variables(device) + self.create_groups(device) def parse(self, inventory, loader, path, cache=True): super(InventoryModule, self).parse(inventory, loader, path) @@ -308,6 +315,7 @@ def parse(self, inventory, loader, path, cache=True): } if token: self.headers.update({"Authorization": "Token %s" % token}) + self.gql_query = self.get_option("query") self.group_by = self.get_option("group_by") self.filters = self.get_option("filters") diff --git a/tests/unit/inventory/test_data/graphql_groups/device_data.json b/tests/unit/inventory/test_data/graphql_groups/device_data.json new file mode 100644 index 00000000..0203b9ae --- /dev/null +++ b/tests/unit/inventory/test_data/graphql_groups/device_data.json @@ -0,0 +1,25 @@ +{ + "name": "mydevice", + "platform": { + "napalm_driver": "asa" + }, + "status": { + "name": "Active" + }, + "primary_ip4": { + "address": "10.10.10.10/32" + }, + "device_role": { + "name": "edge", + "color_category": { + "primary": "red" + } + }, + "site": { + "name": "ATL01" + }, + "tenant": { + "slug": "mytenant", + "type": "local" + } +} \ No newline at end of file diff --git a/tests/unit/inventory/test_graphql.py b/tests/unit/inventory/test_graphql.py new file mode 100644 index 00000000..9ea8f3dc --- /dev/null +++ b/tests/unit/inventory/test_graphql.py @@ -0,0 +1,144 @@ +from __future__ import absolute_import, division, print_function + +__metaclass__ = type + +import pytest +import json +import os + +from functools import partial +from unittest.mock import patch, MagicMock, Mock, call +from ansible.inventory.data import InventoryData +from ansible.errors import AnsibleError +from ansible.utils.display import Display +from netutils.lib_mapper import ANSIBLE_LIB_MAPPER_REVERSE, NAPALM_LIB_MAPPER + +try: + from ansible_collections.networktocode.nautobot.plugins.inventory.gql_inventory import InventoryModule + +except ImportError: + import sys + + # Not installed as a collection + # Try importing relative to root directory of this ansible_modules project + + sys.path.append("plugins/inventory/") + sys.path.append("tests") + from gql_inventory import InventoryModule + + +def load_graphql_device_data(path, test_path): + with open(f"{path}/test_data/{test_path}/device_data.json", "r") as f: + data = json.loads(f.read()) + return data + + +load_relative_test_data = partial(load_graphql_device_data, os.path.dirname(os.path.abspath(__file__))) + + +@pytest.fixture +def inventory_fixture(): + inventory = InventoryModule() + inventory.inventory = InventoryData() + inventory.inventory.add_host("mydevice") + + return inventory + + +@pytest.fixture +def device_data(): + json_data = load_relative_test_data("graphql_groups") + return json_data + + +def test_group_by_path_multiple(inventory_fixture, device_data): + inventory_fixture.group_by = ["device_role.color_category.primary"] + inventory_fixture.create_groups(device_data) + inventory_groups = list(inventory_fixture.inventory.groups.keys()) + local_device_type_inventory_hosts = inventory_fixture.inventory.get_groups_dict().get("red") + assert ["all", "ungrouped", "red"] == inventory_groups + assert ["mydevice"] == local_device_type_inventory_hosts + + +def test_group_by_path(inventory_fixture, device_data): + inventory_fixture.group_by = ["tenant.type"] + inventory_fixture.create_groups(device_data) + inventory_groups = list(inventory_fixture.inventory.groups.keys()) + local_device_type_inventory_hosts = inventory_fixture.inventory.get_groups_dict().get("local") + assert ["all", "ungrouped", "local"] == inventory_groups + assert ["mydevice"] == local_device_type_inventory_hosts + + +def test_group_by_string_only(inventory_fixture, device_data): + inventory_fixture.group_by = ["site"] + inventory_fixture.create_groups(device_data) + inventory_groups = list(inventory_fixture.inventory.groups.keys()) + atl01_inventory_hosts = inventory_fixture.inventory.get_groups_dict().get("ATL01") + assert ["all", "ungrouped", "ATL01"] == inventory_groups + assert ["mydevice"] == atl01_inventory_hosts + + +@patch.object(Display, "display") +def test_no_parent_value(mock_display, inventory_fixture, device_data): + inventory_fixture.group_by = ["color.slug"] + inventory_fixture.create_groups(device_data) + mock_display.assert_any_call("Could not find value for color on device mydevice") + + +@patch.object(Display, "display") +def test_multiple_group_by_one_fail(mock_display, inventory_fixture, device_data): + inventory_fixture.group_by = ["color.slug", "site.name"] + inventory_fixture.create_groups(device_data) + inventory_groups = list(inventory_fixture.inventory.groups.keys()) + atl01_inventory_hosts = inventory_fixture.inventory.get_groups_dict().get("ATL01") + mock_display.assert_any_call("Could not find value for color on device mydevice") + assert ["all", "ungrouped", "ATL01"] == inventory_groups + assert ["mydevice"] == atl01_inventory_hosts + + +def test_multiple_group_by_no_fail(inventory_fixture, device_data): + inventory_fixture.group_by = ["status.name", "site.name"] + inventory_fixture.create_groups(device_data) + inventory_groups = list(inventory_fixture.inventory.groups.keys()) + atl01_inventory_hosts = inventory_fixture.inventory.get_groups_dict().get("ATL01") + active_inventory_hosts = inventory_fixture.inventory.get_groups_dict().get("Active") + assert ["all", "ungrouped", "Active", "ATL01"] == inventory_groups + assert ["mydevice"] == atl01_inventory_hosts + assert ["mydevice"] == active_inventory_hosts + + +@patch.object(Display, "display") +def test_no_chain_value(mock_display, inventory_fixture, device_data): + inventory_fixture.group_by = ["site.type"] + inventory_fixture.create_groups(device_data) + mock_display.assert_any_call("Could not find value for type in site.type on device mydevice.") + + +@patch.object(Display, "display") +def test_no_name_or_slug_value(mock_display, inventory_fixture, device_data): + inventory_fixture.group_by = ["platform"] + inventory_fixture.create_groups(device_data) + mock_display.assert_any_call("No slug or name value for {'napalm_driver': 'asa'} in platform on device mydevice.") + + +@patch.object(Display, "display") +def test_group_name_dict(mock_display, inventory_fixture, device_data): + inventory_fixture.group_by = ["platform"] + inventory_fixture.create_groups(device_data) + mock_display.assert_any_call("No slug or name value for {'napalm_driver': 'asa'} in platform on device mydevice.") + + +def test_add_ipv4(inventory_fixture, device_data): + inventory_fixture.group_by = ["site"] + inventory_fixture.create_groups(device_data) + inventory_fixture.add_ipv4_address(device_data) + mydevice_host = inventory_fixture.inventory.get_host("mydevice") + assert mydevice_host.vars.get("ansible_host") == "10.10.10.10/32" + + +def test_ansible_platform(inventory_fixture, device_data): + inventory_fixture.group_by = ["site"] + inventory_fixture.create_groups(device_data) + inventory_fixture.add_ansible_platform(device_data) + mydevice_host = inventory_fixture.inventory.get_host("mydevice") + assert mydevice_host.vars.get("ansible_network_os") == "cisco.asa.asa"