Merge pull request #287 from quietjoy/issue/110

Feat: AnsibleVarsParser

.gitignore

@@ -22,3 +22,9 @@ dmypy.json
 
 # Vagrant artifacts
 .vagrant/**
+
+# Virtual Environment
+venv/**
+
+# VSCode artifacts
+.vscode/**

ansible_bender/cli.py

@@ -12,7 +12,7 @@
 
 from ansible_bender.api import Application
 from ansible_bender.constants import ANNOTATIONS_KEY, PLAYBOOK_TEMPLATE
-from ansible_bender.core import PbVarsParser
+from ansible_bender.core import AnsibleVarsParser
 from ansible_bender.db import PATH_CANDIDATES
 from ansible_bender.okd import build_inside_openshift
 
@@ -124,6 +124,11 @@ def _do_build_interface(self):
             "--build-user",
             help="the container gets invoked with this user during build"
         )
+        self.build_parser.add_argument(
+            "--inventory",
+            help="path to Ansible inventory",
+            default=None,
+        )
         self.build_parser.add_argument(
             "--build-entrypoint",
             help="the container will be invoked with this entrypoint during the build"
@@ -289,7 +294,7 @@ def _do_clean_interface(self):
         self.lb_parser.set_defaults(subcommand="clean")
 
     def _build(self):
-        pb_vars_p = PbVarsParser(self.args.playbook_path)
+        pb_vars_p = AnsibleVarsParser(self.args.playbook_path, self.args.inventory)
         build, metadata = pb_vars_p.get_build_and_metadata()
         build.metadata = metadata
         if self.args.workdir:

ansible_bender/core.py

@@ -41,6 +41,7 @@
 import tempfile
 from pathlib import Path
 import uuid
+import configparser
 
 import jsonschema
 import yaml
@@ -54,6 +55,11 @@
 from ansible_bender.utils import run_cmd, ap_command_exists, random_str, graceful_get, \
     is_ansibles_python_2
 
+from ansible.inventory.manager import InventoryManager
+from ansible.vars.manager import VariableManager
+from ansible.parsing.dataloader import DataLoader
+from ansible_bender.conf import Build, ImageMetadata
+
 logger = logging.getLogger(__name__)
 # callback_whitelist got renamed to callbacks_enabled in ansible
 # drop callback_whitelist once 2.15 is released
@@ -389,3 +395,221 @@ def get_build_and_metadata(self):
         self.process_pb_vars(bender_data)
 
         return self.build, self.metadata
+
+
+class AnsibleVarsParser:
+    """
+    AnsibleVarParser is used to parse the ansible variables from the playbook and inventory
+    This class attempts to use the ansible API to parse the variables
+    If the inventory is not specified, the class attempts to find the inventory file
+    If the inventory is not found, the class will fall back to PbVarsParser to parse the variables
+    """
+
+    # Prefix for ansible_bender variables
+    _variable_prefix = "ansible_bender"
+
+    _inventory_path = None
+
+    _playbook_path = None
+
+    _loader: DataLoader = None
+
+    _inventory: InventoryManager = None
+
+    _variable_manager: VariableManager = None
+
+    _hosts_in_playbook: list = None
+
+    _hosts: dict = {}
+
+    build: Build = None
+
+    metadata: ImageMetadata = None
+
+    def __init__(self, playbook_path, inventory_path = None) -> None:
+        self._base_dir = os.path.dirname(playbook_path)
+        self._playbook_path = playbook_path
+        self._inventory_path = inventory_path if inventory_path else self._find_inventory_path()
+        self.build = Build()
+        self.metadata = ImageMetadata()
+        self.build.metadata = self.metadata
+
+    def __getitem__(self, key):
+        return self._hosts[key]
+
+    def _find_ansible_cfg(self) -> str:
+        """
+        Find the ansible.cfg file
+        Precendence: https://docs.ansible.com/ansible/latest/reference_appendices/config.html
+        """
+        # Check environment variable
+        if "ANSIBLE_CONFIG" in os.environ:
+            return os.environ["ANSIBLE_CONFIG"]
+
+        # Check in self._base_dir (playbook directory)
+        if os.path.isfile(os.path.join(self._base_dir, "ansible.cfg")):
+            return os.path.join(self._base_dir, "ansible.cfg")
+
+        # Check in home directory
+        if os.path.isfile(os.path.join(os.path.expanduser("~"), ".ansible.cfg")):
+            return os.path.join(os.path.expanduser("~"), ".ansible.cfg")
+
+        # Check in /etc/ansible/ansible.cfg
+        if os.path.isfile("/etc/ansible/ansible.cfg"):
+            return "/etc/ansible/ansible.cfg"
+
+        # No ansible.cfg file found
+        return None
+
+    def _find_inventory_path(self) -> str:
+        """
+        Get the inventory path from either
+        1. ansible.cfg
+        2. the default ansible inventory location (/etc/ansible/hosts)
+        """
+        ansible_cfg_path = self._find_ansible_cfg()
+
+        if ansible_cfg_path:
+            return self._get_inventory_from_ansible_cfg(ansible_cfg_path)
+        else:
+            if os.path.isfile("/etc/ansible/hosts"):
+                return"/etc/ansible/hosts"
+
+        return None
+
+    def _get_inventory_from_ansible_cfg(self, ansible_cfg_path) -> str:
+        """
+        Get the inventory path from the ansible.cfg file
+        """
+        config = configparser.ConfigParser()
+        config.read(ansible_cfg_path)
+        inv_path = config.get("defaults", "inventory", fallback=None)
+
+        # Possible to have an ansible.cfg file without an inventory path
+        if inv_path is None:
+            return None
+
+        # Inventory path can be absolute or relative in ansible.cfg
+        if os.path.isabs(inv_path):
+            return inv_path
+        else:
+            return os.path.join(os.path.dirname(ansible_cfg_path), inv_path)
+
+    def _get_host_names(self, hosts) -> list:
+        """
+        Parse the hosts from the playbook
+        """
+        parsed_host_information = []
+
+        for host in hosts:
+            parsed_host_information.append(host.get("hosts"))
+
+        return parsed_host_information
+
+    def _extract_variables(self, ansible_variables, variables):
+        """
+        Convinience function to get the variables from the variables dictionary
+        Gets the ansible_bender variables and the prepended variables
+        """
+
+        if variables is None:
+            return ansible_variables
+
+        for key, value in variables.items():
+            if str(key).startswith(self._variable_prefix):
+                    # Assume that ansible API has already 
+                    # pulled variables with correct precedence
+                new_key = key.replace(self._variable_prefix, "")
+
+                    # key can be prepended with ansible_bender_ or ansible_bender
+                    # if it is prepended with ansible_bender_, remove the _
+                if new_key.startswith("_"):
+                    new_key = new_key[1:]
+
+                    # If the new_key is blank
+                    # then the "ansible_bender" variable was used
+                    # Add the variables to the ansible_variables
+                if new_key == "":
+                    ansible_variables.update(value)
+                else:
+                    ansible_variables[new_key] = value
+
+        return ansible_variables
+
+    def _get_vars_from_playbook(self, host_name) -> dict:
+        """
+        Parse the playbook variables related to the ansible_bender information
+        """
+        playbook_variables = {}
+
+        try:
+            for host in self._hosts_in_playbook:
+                if host.get("hosts") == host_name:
+                    playbook_variables = self._extract_variables(playbook_variables, host.get("vars"))
+        except Exception as e:
+            print("Error getting playbook variables for host: " + host_name + " - " + str(e))
+            pass
+
+        return playbook_variables
+
+    def _get_vars_for_host(self, host_name) -> dict:
+        """
+        Parse the host variables from the inventory information
+        """
+        ansible_variables = {}
+        try:
+            ansible_host = self._inventory.get_host(host_name)
+            all_vars_for_host = self._variable_manager.get_vars(host=ansible_host, 
+                                                            include_hostvars=True, 
+                                                            include_delegate_to=True)
+
+            # find prepended variables
+            self._extract_variables(ansible_variables, all_vars_for_host)                
+        except Exception as e:
+            print("Error getting host and group vars for host: " + host_name + " - " + str(e))
+            pass
+
+        # Append the playbook variables to the ansible variables
+        ansible_variables.update(self._get_vars_from_playbook(host_name))
+
+        return ansible_variables
+
+
+    def get_build_and_metadata(self):
+        """
+        Find all hosts in the playbook and get their variables
+        return the build and imagemetadata information
+        """
+        if self._inventory_path:
+            # Ansible API objects used to pull variables
+            relative_playbook_file_path = os.path.basename(self._playbook_path)
+            self._loader = DataLoader()
+            self._loader.set_basedir(self._base_dir)
+            self._inventory = InventoryManager(loader=self._loader, sources=self._inventory_path)
+            self._variable_manager = VariableManager(loader=self._loader, inventory=self._inventory)
+            self._hosts_in_playbook = self._loader.load_from_file(relative_playbook_file_path)
+
+            host_names = self._get_host_names(self._hosts_in_playbook)
+            for host_name in host_names:
+                ansible_vars = self._get_vars_for_host(host_name)
+                self._hosts[host_name] = ansible_vars
+
+            # We only support one host in the playbook
+            host = self._hosts[list(self._hosts)[0]] if len(self._hosts) > 0 else None
+
+            if not host:
+                raise Exception("No hosts found in the playbook")
+
+            self.metadata.update_from_configuration(host.get("target_image", {}))
+            self.build.update_from_configuration(host)
+        else:
+            # At this point
+            # 1. No inventory was provided
+            # 2. No ansible.cfg file was found with inventory information
+            # 3. No default inventory file was found
+            # So forget about using the ansible API to get variables
+            # Just manually parse the playbook
+            pb_vars_p = PbVarsParser(self._playbook_path)
+            self.build, self.metadata = pb_vars_p.get_build_and_metadata()
+
+        return self.build, self.metadata

docs/md_docs/configuration.md

@@ -5,19 +5,89 @@
 With dockerfiles, this is being usually done with instructions such as `LABEL`,
 `ENV` or `EXPOSE`. Bender supports two ways of configuring the metadata:
 
-* Setting specific Ansible variables inside your playbook.
-* CLI options of `ansible-bender build`.
+* Setting specific Ansible variables
+* CLI options of `ansible-bender build`
 
 
-### Via playbook vars
+### Via ansible vars
 
-Configuration is done using a top-level Ansible variable `ansible_bender`. All
-the values are nested under it. The values are processed before a build starts.
-The changes to values are not reflected during a playbook run.
+Configuration can be done using ansible variables. Variables can be specified in 
+the playbook, host_vars or group_vars. If using host or group vars, you also
+need to specify the inventory for your ansible project, either through cli
+parameter, `--inventory`, or in `ansible.cfg`.
 
-If your playbook has multiple plays, the `ansible_bender` variable is processed
-only from the first play. All the plays will end up in a single container image.
+The values are processed before a build starts. The changes to values are not reflected 
+during a playbook run. If your playbook has multiple plays, the `ansible_bender` variable is processed only from the first play.
 
+The `ansible_bender` variables can be specified in one of two ways:
+
+1. a top-level Ansible variable `ansible_bender` with all the values nested under it 
+1. prepended with `ansible_bender_*`. This is convenient if you want to split your variables across several files.
+For example, you have a group of hosts with a common base container. You could specify the base container
+in the groups `group_vars` file and the host specific container information in the `host_vars` file.
+
+#### Top level ansible_bender variable
+
+The ansible bender configuration variables can specified under a single variable.
+For example,
+
+##### playbook.yml
+```
+---
+- name: Single variable configuration
+  hosts: all
+  vars:
+    ansible_bender:
+      base_image: docker.io/python:3-alpine
+
+      working_container:
+        volumes:
+          - '{{ playbook_dir }}:/src:Z'
+
+      target_image:
+        name: a-very-nice-image
+        working_dir: /src
+        environment:
+          FILE_TO_PROCESS: README.md
+```
+
+#### Prepended variables
+
+Variables can also be split out amoung different files.
+
+In this example, there are two hosts in the inventory, belonging to a single group.
+
+##### inventory.yml
+```yaml
+[group1]
+host1
+host2
+```
+
+You can define a common base container in
+the `group_vars` file using the `base_image` variable prepended with `ansible_bender_`
+
+##### group_vars/group1
+```
+ansible_bender_base_image: docker.io/python:3-alpine
+```
+
+Next you can define host specific ansible builder variables in the host_vars
+
+##### host_vars/host1
+```
+ansible_bender_target_image:
+  name: host_var_host1
+  working_dir: /tmp
+  environment:
+    FILE_TO_PROCESS: README.md
+```
+
+There are three variables that can be prepended:
+
+1. ansible_bender_base_image
+1. ansible_bender_target_image
+1. ansible_bender_working_container
 
 #### Top-level keys