diff --git a/docs/source/deploying-volttron/platform-configuration.rst b/docs/source/deploying-volttron/platform-configuration.rst index 569d86485c..b3b9f5f390 100644 --- a/docs/source/deploying-volttron/platform-configuration.rst +++ b/docs/source/deploying-volttron/platform-configuration.rst @@ -6,11 +6,11 @@ Platform Configuration Each instance of the VOLTTRON platform includes a `config` file which is used to configure the platform instance on startup. This file is kept in :term:`VOLTTRON_HOME` and is created using the `volttron-cfg` (`vcfg`) command, or will -be created with default values on start up of the platform otherwise. +be created with default values on start up of the platform otherwise. `vcfg` also provides commands to configure and +install few frequently used agents and update configuration-store entries Following is helpful information about the `config` file and the `vcfg` command. - VOLTTRON_HOME ============= @@ -75,9 +75,13 @@ The example consists of the following entries: VOLTTRON Config =============== -The `volttron-cfg` or `vcfg` command allows for an easy configuration of the VOLTTRON environment. The command includes -the ability to set up the platform configuration, an instance of the platform historian, VOLTTRON Central UI, and -VOLTTRON Central Platform agent. +The `volttron-cfg` or `vcfg` command allows for an easy configuration of the VOLTTRON environment. +The `vcfg` command can be run without any arguments to go through all available configuration steps sequentially and +pick what is appropriate for a given instance of VOLTTRON. This is useful when starting with a new VOLTTRON instance. + +`vcfg` command can also be used with specific subcommands to configure and install specific agents such as +listener agent, platform historian, VOLTTRON Central UI, and VOLTTRON Central Platform agent +or add one or more agent configurations to VOLTTRON's configuration store. Running `vcfg` will create a `config` file in `VOLTTRON_HOME` which will be populated according to the answers to prompts. This process should be repeated for each platform instance, and can be re-run to reconfigure a platform @@ -158,12 +162,52 @@ Optional Arguments - **-v, --verbose** - Enables verbose output in standard-output (PIP output, etc.) - **--vhome VHOME** - Provide a path to set `VOLTTRON_HOME` for this instance - **--instance-name INSTANCE_NAME** - Provide a name for this instance. Required for running secure agents mode - - **--list-agents** - Display a list of configurable agents (Listener, Platform Driver, Platform Historian, VOLTTRON - Central, VOLTTRON Central Platform) - - **--agent AGENT [AGENT ...]** - Configure listed agents - **--agent-isolation-mode** - Require that agents run as their own Unix users (this requires running `scripts/secure_user_permissions.sh` as `sudo`) +Sub commands +------------ +**--list-agents** +~~~~~~~~~~~~~~~~~~ + Display a list of configurable agents (Listener, Platform Driver, Platform Historian, VOLTTRON + Central, VOLTTRON Central Platform) + +**--agent AGENT [AGENT ...]** +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Configure and install one of the listed agents for a specific VOLTTRON instance + +**update-config-store --metadata-file METADATA_FILE_OR_DIR [METADATA_FILE_OR_DIR ...]** +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Command to bulk add/update multiple agent configurations to VOLTTRON's configuration store. This is especially useful +for automated deployment use cases. For example, a automated deployment setup where devices in a network are +detected, and configurations for platform driver and control agents such as ILCAgent are auto generated using semantic +tags. In such as scenario, adding all the generated configurations to the configuration store using the below command +is more efficient than running the "vctl config store" command once for each configuration + +This command can only be used when volttron instance is not running. + +Usage: + +.. code-block:: bash + + vcfg update-config-store --metadata-file + + +Format for Metadata file: + +.. code-block:: + + { "vip-id": [ + { + "config-name": "optional. name. defaults to config + "config": "json config or string config or config file name", + "config-type": "optional. type of config - csv or json or raw. defaults to json" + }, ... + ],... + } + + RabbitMQ Arguments ------------------ vcfg command to configure a single RabbitMQ instance of VOLTTRON. diff --git a/volttron/platform/agent/utils.py b/volttron/platform/agent/utils.py index 59fbc694a0..cda9c8cf7f 100644 --- a/volttron/platform/agent/utils.py +++ b/volttron/platform/agent/utils.py @@ -153,7 +153,7 @@ def load_config(config_path): if not os.path.exists(config_path): raise ValueError( - f"Config file specified by AGENT_CONFIG path {config_path} does not exist." + f"Config file specified by path {config_path} does not exist." ) # First attempt parsing the file with a yaml parser (allows comments natively) @@ -169,7 +169,7 @@ def load_config(config_path): with open(config_path) as f: return parse_json_config(f.read()) except Exception as e: - _log.error("Problem parsing agent configuration") + _log.error(f"Problem parsing configuration {config_path}: {e}") raise diff --git a/volttron/platform/instance_setup.py b/volttron/platform/instance_setup.py index d8e91a3de1..83066aa24c 100644 --- a/volttron/platform/instance_setup.py +++ b/volttron/platform/instance_setup.py @@ -37,6 +37,7 @@ # }}} import argparse import hashlib +import json import os import sys import tempfile @@ -46,6 +47,7 @@ from shutil import copy from urllib.parse import urlparse import logging +from argparse import RawTextHelpFormatter from gevent import subprocess from gevent.subprocess import Popen @@ -57,10 +59,13 @@ from volttron.platform import jsonapi from volttron.platform.agent.known_identities import PLATFORM_WEB, PLATFORM_DRIVER, VOLTTRON_CENTRAL from volttron.platform.agent.utils import get_platform_instance_name, wait_for_volttron_startup, \ - is_volttron_running, wait_for_volttron_shutdown, setup_logging + is_volttron_running, wait_for_volttron_shutdown, setup_logging, format_timestamp, get_aware_utc_now, \ + parse_json_config from volttron.utils import get_hostname from volttron.utils.prompt import prompt_response, y, n, y_or_n from . import get_home, get_services_core, set_home +from volttron.platform.agent.utils import load_config as load_yml_or_json +from volttron.platform.store import process_raw_config if is_rabbitmq_available(): from bootstrap import install_rabbit, default_rmq_dir @@ -79,6 +84,7 @@ # Determines if VOLTTRON instance can remain running with vcfg use_active = "N" + def _load_config(): """Loads the config file if the path exists.""" path = os.path.join(get_home(), 'config') @@ -155,24 +161,24 @@ def _is_bound_already(address): return already_bound -def fail_if_instance_running(args): +def fail_if_instance_running(message, prompt=True): home = get_home() if os.path.exists(home) and\ is_volttron_running(home): - global use_active - use_active = prompt_response( - "The VOLTTRON Instance is currently running. " - "Installing agents to an active instance may overwrite currently installed " - "and active agents on the platform, causing undesirable behavior. " - "Would you like to continue?", - valid_answers=y_or_n, - default='Y') - if use_active in y: - return + if prompt: + global use_active + use_active = prompt_response( + message + + "Would you like to continue?", + valid_answers=y_or_n, + default='Y') + if use_active in y: + return else: - print(""" + print(message) + print(""" Please execute: volttron-ctl shutdown --platform @@ -180,7 +186,7 @@ def fail_if_instance_running(args): to stop the instance. """) - sys.exit() + sys.exit(1) def fail_if_not_in_src_root(): @@ -189,7 +195,7 @@ def fail_if_not_in_src_root(): print(""" volttron-cfg needs to be run from the volttron top level source directory. """) - sys.exit() + sys.exit(1) def _start_platform(): @@ -261,13 +267,14 @@ def _is_agent_installed(tag): def installs(agent_dir, tag, identity=None, post_install_func=None): def wrap(config_func): global available_agents + def func(*args, **kwargs): global use_active print('Configuring {}.'.format(agent_dir)) config = config_func(*args, **kwargs) _update_config_file() - #TODO: Optimize long vcfg install times - #TODO: (potentially only starting the platform once per vcfg) + # TODO: Optimize long vcfg install times + # TODO: (potentially only starting the platform once per vcfg) if use_active in n: _start_platform() @@ -375,6 +382,7 @@ def set_dependencies(requirement): subprocess.check_call(cmds) return + def _create_web_certs(): global config_opts """ @@ -403,7 +411,7 @@ def _create_web_certs(): prompt = '\tOrganization:' cert_data['organization'] = prompt_response(prompt, mandatory=True) prompt = '\tOrganization Unit:' - cert_data['organization-unit'] = prompt_response(prompt,mandatory=True) + cert_data['organization-unit'] = prompt_response(prompt, mandatory=True) cert_data['common-name'] = get_platform_instance_name() + '-root-ca' data = {'C': cert_data.get('country'), 'ST': cert_data.get('state'), @@ -412,12 +420,13 @@ def _create_web_certs(): 'OU': cert_data.get('organization-unit'), 'CN': cert_data.get('common-name')} crts.create_root_ca(overwrite=False, **data) - copy(crts.cert_file(crts.root_ca_name),crts.cert_file(crts.trusted_ca_name)) + copy(crts.cert_file(crts.root_ca_name), crts.cert_file(crts.trusted_ca_name)) else: return 1 print("Creating new web server certificate.") - crts.create_signed_cert_files(name=PLATFORM_WEB + "-server", cert_type='server', ca_name=crts.root_ca_name, fqdn=get_hostname()) + crts.create_signed_cert_files(name=PLATFORM_WEB + "-server", cert_type='server', ca_name=crts.root_ca_name, + fqdn=get_hostname()) return 0 @@ -527,13 +536,15 @@ def do_instance_name(): instance_name = new_instance_name config_opts['instance-name'] = '"{}"'.format(instance_name) + def do_web_enabled_rmq(vhome): global config_opts # Full implies that it will have a port on it as well. Though if it's # not in the address that means that we haven't set it up before. - full_bind_web_address = config_opts.get('bind-web-address', - 'https://' + get_hostname()) + full_bind_web_address = config_opts.get( + 'bind-web-address', + 'https://' + get_hostname()) parsed = urlparse(full_bind_web_address) @@ -575,11 +586,9 @@ def do_web_enabled_rmq(vhome): def do_web_enabled_zmq(vhome): global config_opts - # Full implies that it will have a port on it as well. Though if it's # not in the address that means that we haven't set it up before. - full_bind_web_address = config_opts.get('bind-web-address', - 'https://' + get_hostname()) + full_bind_web_address = config_opts.get('bind-web-address', 'https://' + get_hostname()) parsed = urlparse(full_bind_web_address) @@ -698,7 +707,7 @@ def get_cert_and_key(vhome): try: if certs.Certs.validate_key_pair(platform_web_cert, platform_web_key): print('\nThe following certificate and keyfile exists for web access over https: \n{}\n{}'.format( - platform_web_cert,platform_web_key)) + platform_web_cert, platform_web_key)) prompt = '\nDo you want to use these certificates for the web server?' if prompt_response(prompt, valid_answers=y_or_n, default='Y') in y: config_opts['web-ssl-cert'] = platform_web_cert @@ -713,8 +722,6 @@ def get_cert_and_key(vhome): print(e) pass - - # Either are there no valid existing certs or user decided to overwrite the existing file. # Prompt for new files while cert_error: @@ -753,10 +760,8 @@ def get_cert_and_key(vhome): else: cert_error = _create_web_certs() if not cert_error: - platform_web_cert = os.path.join(vhome, 'certificates/certs/', - PLATFORM_WEB+"-server.crt") - platform_web_key = os.path.join(vhome, 'certificates/private/', - PLATFORM_WEB + "-server.pem") + platform_web_cert = os.path.join(vhome, 'certificates/certs/', PLATFORM_WEB+"-server.crt") + platform_web_key = os.path.join(vhome, 'certificates/private/', PLATFORM_WEB + "-server.pem") config_opts['web-ssl-cert'] = platform_web_cert config_opts['web-ssl-key'] = platform_web_key @@ -793,8 +798,7 @@ def do_vcp(): except KeyError: vc_address = config_opts.get('volttron-central-address', - config_opts.get('bind-web-address', - 'https://' + get_hostname())) + config_opts.get('bind-web-address', 'https://' + get_hostname())) if not is_vc: parsed = urlparse(vc_address) address_only = vc_address @@ -892,7 +896,6 @@ def wizard(): # Start true configuration here. volttron_home = get_home() - confirm_volttron_home() _load_config() _update_config_file() if use_active in n: @@ -965,8 +968,7 @@ def wizard(): prompt = 'Will this instance be controlled by volttron central?' response = prompt_response(prompt, valid_answers=y_or_n, default='Y') if response in y: - if not _check_dependencies_met( - "drivers") or not _check_dependencies_met("web"): + if not _check_dependencies_met("drivers") or not _check_dependencies_met("web"): print("VCP dependencies not installed. Installing now...") if not _check_dependencies_met("drivers"): set_dependencies("drivers") @@ -993,11 +995,137 @@ def wizard(): if response in y: do_listener() + +def read_agent_configs_from_store(store_source, path=True): + if path: + with open(store_source) as f: + store = parse_json_config(f.read()) + else: + store = store_source + return store + + +def update_configs_in_store(args_dict): + + vhome = get_home() + metadata_files = list() + + args_list = args_dict['metadata_file'] + # validate args + for item in args_list: + if os.path.isdir(item): + for f in os.listdir(item): + file_path = os.path.join(item, f) + if os.path.isfile(file_path): + metadata_files.append(file_path) + elif os.path.isfile(item): + metadata_files.append(item) + else: + print(f"Value is neither a file nor a directory: {args_dict['metadata_file']}: ") + print(f"The --metadata-file accepts one or more metadata files or directory containing metadata file") + _exit_with_metadata_error() + + # Validate each file content and load config + for metadata_file in metadata_files: + metadata_dict = dict() + try: + metadata_dict = load_yml_or_json(metadata_file) + except Exception as e: + print(f"Invalid metadata file: {metadata_file}: {e}") + exit(1) + + for vip_id in metadata_dict: + configs = metadata_dict[vip_id] + if isinstance(configs, dict): + # only single config for this vip id + configs = [configs] + if not isinstance(configs, list): + print( + f"Metadata for vip-identity {vip_id} in file {metadata_file} " + f"should be a dictionary or list of dictionary. " + f"Got type {type(configs)}") + _exit_with_metadata_error() + + configs_updated = False + agent_store_path = os.path.join(vhome, "configuration_store", vip_id+".store") + if os.path.isfile(agent_store_path): + # load current store configs as python object for comparison + store_configs = read_agent_configs_from_store(agent_store_path) + else: + store_configs = dict() + + for config_dict in configs: + if not isinstance(config_dict, dict): + print(f"Metadata for vip-identity {vip_id} in file {metadata_file} " + f"should be a dictionary or list of dictionary. " + f"Got type {type(config_dict)}") + _exit_with_metadata_error() + + config_name = config_dict.get("config-name", "config") + config_type = config_dict.get("config-type", "json") + config = config_dict.get("config") + if config is None: + print(f"No config entry found in file {metadata_file} for vip-id {vip_id} and " + f"config-name {config_name}") + _exit_with_metadata_error() + + # If there is config validate it + # Check if config is file path + if isinstance(config, str) and os.path.isfile(config): + raw_data = open(config).read() + # try loading it into appropriate python object to validate if file content and config-type match + processed_data = process_raw_config(raw_data, config_type) + elif isinstance(config, str) and config_type == 'raw': + raw_data = config + processed_data = config + else: + if not isinstance(config, (list, dict)): + processed_data = raw_data = None + print('Value for key "config" should be one of the following: \n' + '1. filepath \n' + '2. string with "config-type" set to "raw" \n' + '3. a dictionary \n' + '4. list ') + _exit_with_metadata_error() + else: + processed_data = config + raw_data = jsonapi.dumps(processed_data) + + current = store_configs.get(config_name) + + if not current or process_raw_config(current.get('data'), current.get('type')) != processed_data: + store_configs[config_name] = dict() + store_configs[config_name]['data'] = raw_data + store_configs[config_name]['type'] = config_type + store_configs[config_name]['modified'] = format_timestamp(get_aware_utc_now()) + configs_updated = True + + # All configs processed for current vip-id + # if there were updates write the new configs to file + if configs_updated: + os.makedirs(os.path.dirname(agent_store_path), exist_ok=True) + with open(agent_store_path, 'w+') as f: + json.dump(store_configs, f) + + +def _exit_with_metadata_error(): + print(""" +Metadata file format: +{ "vip-id": [ + { + "config-name": "optional. name. defaults to config + "config": "json config or string config or config file name", + "config-type": "optional. type of config - csv or json or raw. defaults to json" + }, ... + ],... +}""") + exit(1) + + def process_rmq_inputs(args_dict, instance_name=None): - #print(f"args_dict:{args_dict}, args") if not is_rabbitmq_available(): raise RuntimeError("Rabbitmq Dependencies not installed please run python bootstrap.py --rabbitmq") - confirm_volttron_home() + vhome = get_home() if args_dict['installation-type'] in ['federation', 'shovel'] and not _check_dependencies_met('web'): @@ -1042,9 +1170,11 @@ def process_rmq_inputs(args_dict, instance_name=None): else: print("Invalid installation type. Acceptable values single|federation|shovel") sys.exit(1) - setup_rabbitmq_volttron(args_dict['installation-type'], verbose, instance_name=instance_name, max_retries=args_dict['max_retries']) + setup_rabbitmq_volttron(args_dict['installation-type'], verbose, instance_name=instance_name, + max_retries=args_dict['max_retries']) else: - setup_rabbitmq_volttron(args_dict['installation-type'], verbose, prompt=True, instance_name=instance_name, max_retries=args_dict['max_retries']) + setup_rabbitmq_volttron(args_dict['installation-type'], verbose, prompt=True, instance_name=instance_name, + max_retries=args_dict['max_retries']) def main(): @@ -1055,34 +1185,60 @@ def main(): parser.add_argument('--vhome', help="Path to volttron home") parser.add_argument('--instance-name', dest='instance_name', help="Name of this volttron instance") parser.set_defaults(is_rabbitmq=False) + parser.set_defaults(config_update=False) group = parser.add_mutually_exclusive_group() agent_list = '\n\t' + '\n\t'.join(sorted(available_agents.keys())) group.add_argument('--list-agents', action='store_true', dest='list_agents', help='list configurable agents{}'.format(agent_list)) - rabbitmq_parser = parser.add_subparsers(title='rabbitmq', - metavar='', - dest='parser_name') - single_parser = rabbitmq_parser.add_parser('rabbitmq', help='Configure rabbitmq for single instance, ' - 'federation, or shovel either based on ' - 'configuration file in yml format or providing ' - 'details when prompted. \nUsage: vcfg rabbitmq ' - 'single|federation|shovel --config --max-retries ]') - single_parser.add_argument('installation-type', default='single', help='Rabbitmq option for installation. Installation type can be single|federation|shovel') + subparsers = parser.add_subparsers(dest="cmd") + single_parser = subparsers.add_parser('rabbitmq', help='Configure rabbitmq for single instance, ' + 'federation, or shovel either based on ' + 'configuration file in yml format or providing ' + 'details when prompted. \nUsage: vcfg rabbitmq ' + 'single|federation|shovel --config --max-retries ]') + single_parser.add_argument('installation-type', default='single', + help='Rabbitmq option for installation. ' + 'Installation type can be single|federation|shovel') single_parser.add_argument('--max-retries', help='Optional Max retry attempt', type=int, default=12) single_parser.add_argument('--config', help='Optional path to rabbitmq config yml', type=str) single_parser.set_defaults(is_rabbitmq=True) group.add_argument('--agent', nargs='+', - help='configure listed agents') + help='configure listed agents') group.add_argument('--agent-isolation-mode', action='store_true', dest='agent_isolation_mode', help='Require that agents run with their own users (this requires running ' 'scripts/secure_user_permissions.sh as sudo)') + config_store_parser = subparsers.add_parser("update-config-store", formatter_class=RawTextHelpFormatter, + help="Update one or more config entries for one more agents") + config_store_parser.set_defaults(config_update=True) + # start with just a metadata file support. + # todo - add support vip-id, directory + # vip-id, file with multiple configs etc. + # config_arg_group = config_store_parser.add_mutually_exclusive_group() + # meta_group = config_arg_group.add_mutually_exclusive_group() + config_store_parser.add_argument('--metadata-file', required=True, nargs='+', + help="""One or more metadata file or directory containing metadata file, +where each metadata file contain details of configs for one or more agent instance +Metadata file format: +{ "vip-id": [ + { + "config-name": "optional. name. defaults to config + "config": "json config or string config or config file name", + "config-type": "optional. type of config - csv or json or raw. defaults to json" + }, ... + ],... +}""") + + # single_agent_group = config_arg_group.add_mutually_exclusive_group() + # single_agent_group.add_argument("--vip-id", + # help='vip-identity of the agent for which config store should be updated') + # single_agent_group.add_argument("--config-path", + # help="json file containing configs or directory containing config files") args = parser.parse_args() - verbose = args.verbose # Protect against configuration of base logger when not the "main entry point" if verbose: @@ -1094,8 +1250,18 @@ def main(): if args.vhome: set_home(args.vhome) prompt_vhome = False + + confirm_volttron_home() # if not args.rabbitmq or args.rabbitmq[0] in ["single"]: - fail_if_instance_running(args) + if args.agent: + message = "The VOLTTRON Instance is currently running. " \ + "Installing agents to an active instance may overwrite currently installed "\ + "and active agents on the platform, causing undesirable behavior. " + fail_if_instance_running(message) + if args.config_update: + message = f"VOLTTRON is running using at {get_home()}, " \ + "you can add/update single configuration using vctl config command." + fail_if_instance_running(message, prompt=False) fail_if_not_in_src_root() if use_active in n: atexit.register(_cleanup_on_exit) @@ -1110,6 +1276,8 @@ def main(): _update_config_file() elif args.is_rabbitmq: process_rmq_inputs(vars(args)) + elif args.config_update: + update_configs_in_store(vars(args)) elif not args.agent: wizard() @@ -1121,8 +1289,6 @@ def main(): print('"{}" not configurable with this tool'.format(agent)) else: valid_agents = True - if valid_agents: - confirm_volttron_home() # Configure agents for agent in args.agent: diff --git a/volttrontesting/platform/test_vcfg_config_update.py b/volttrontesting/platform/test_vcfg_config_update.py new file mode 100644 index 0000000000..c34fdfa652 --- /dev/null +++ b/volttrontesting/platform/test_vcfg_config_update.py @@ -0,0 +1,467 @@ +import json + +import pytest +import os +import shutil +import subprocess +from csv import DictReader +from io import StringIO + +from volttrontesting.utils.platformwrapper import create_volttron_home +from volttron.platform.agent.utils import parse_json_config +from volttrontesting.fixtures.volttron_platform_fixtures import build_wrapper, cleanup_wrapper +from volttrontesting.utils.utils import get_rand_vip + +METATADATA_FILE_FORMAT = """Metadata file format: +{ "vip-id": [ + { + "config-name": "optional. name. defaults to config + "config": "json config or string config or config file name", + "config-type": "optional. type of config - csv or json or raw. defaults to json" + }, ... + ],... +}""" + + +@pytest.fixture(scope="module") +def shared_vhome(): + debug_flag = os.environ.get('DEBUG', False) + vhome = create_volttron_home() + yield vhome + if not debug_flag: + shutil.rmtree(vhome, ignore_errors=True) + + +@pytest.fixture() +def vhome(): + debug_flag = os.environ.get('DEBUG', False) + vhome = create_volttron_home() + yield vhome + if not debug_flag: + shutil.rmtree(vhome, ignore_errors=True) + + +@pytest.fixture(scope="module") +def single_vinstance(): + address = get_rand_vip() + wrapper = build_wrapper(address, + messagebus='zmq', + ssl_auth=False, + auth_enabled=False) + yield wrapper + cleanup_wrapper(wrapper) + + +# Only integration test. Rest are unit tests +def test_fail_if_volttron_is_running(single_vinstance, monkeypatch): + monkeypatch.setenv("VOLTTRON_HOME", single_vinstance.volttron_home) + process = subprocess.run(["vcfg", "--vhome", single_vinstance.volttron_home, + "update-config-store", "--metadata-file", "test"], + env=os.environ, + cwd=os.environ.get("VOLTTRON_ROOT"), + stderr=subprocess.PIPE, + stdout=subprocess.PIPE + ) + assert process.stdout.decode("utf-8").startswith( + f"VOLTTRON is running using at {single_vinstance.volttron_home}, " + f"you can add/update single configuration using vctl config command.") + assert process.returncode == 1 + + +def test_help(monkeypatch, shared_vhome): + monkeypatch.setenv("VOLTTRON_HOME", shared_vhome) + process = subprocess.run(["vcfg", "--vhome", shared_vhome, "update-config-store", "--help"], + env=os.environ, + cwd=os.environ.get("VOLTTRON_ROOT"), + stderr=subprocess.PIPE, + stdout=subprocess.PIPE + ) + assert process.stdout.startswith(b"usage: vcfg update-config-store [-h] --metadata-file METADATA_FILE") + + +def test_no_arg(monkeypatch, shared_vhome): + monkeypatch.setenv("VOLTTRON_HOME", shared_vhome) + process = subprocess.run(["vcfg", "--vhome", shared_vhome, "update-config-store"], + env=os.environ, + cwd=os.environ.get("VOLTTRON_ROOT"), + stderr=subprocess.PIPE, + stdout=subprocess.PIPE + ) + assert process.stderr.startswith(b"usage: vcfg update-config-store [-h] --metadata-file METADATA_FILE") + + +def test_no_args_value(monkeypatch, shared_vhome): + monkeypatch.setenv("VOLTTRON_HOME", shared_vhome) + process = subprocess.run(["vcfg", "--vhome", shared_vhome, "update-config-store", "--metadata-file"], + env=os.environ, + cwd=os.environ.get("VOLTTRON_ROOT"), + stderr=subprocess.PIPE, + stdout=subprocess.PIPE + ) + assert process.stderr.startswith(b"usage: vcfg update-config-store [-h] --metadata-file METADATA_FILE") + + +def test_invalid_file_path(monkeypatch, shared_vhome): + monkeypatch.setenv("VOLTTRON_HOME", shared_vhome) + process = subprocess.run(["vcfg", "--vhome", shared_vhome, "update-config-store", "--metadata-file", "invalid"], + env=os.environ, + cwd=os.environ.get("VOLTTRON_ROOT"), + stderr=subprocess.PIPE, + stdout=subprocess.PIPE + ) + expected_message = "Value is neither a file nor a directory: ['invalid']: \n" \ + "The --metadata-file accepts one or more metadata " \ + "files or directory containing metadata file\n\n" + METATADATA_FILE_FORMAT + + assert process.stdout.decode('utf-8').strip() == expected_message + assert process.returncode == 1 + + +@pytest.mark.parametrize('json_metadata, json_file, config_name', [ + ({"vip-id-1": {}}, "no_config_json1", "config"), + ({"vip-id-1": [{"config-name": "config"}]}, "no_config_json2", "config"), + ({"vip-id-1": [{"config-name": "new_config", "config_type": "json"}]}, "no_config_json2", "new_config") +]) +def test_no_config_metadata(monkeypatch, vhome, json_metadata, json_file, config_name): + monkeypatch.setenv("VOLTTRON_HOME", vhome) + file_path = os.path.join(vhome, json_file) + with open(file_path, "w") as f: + f.write(json.dumps(json_metadata)) + expected_message = f"No config entry found in file {file_path} for vip-id vip-id-1 and " \ + f"config-name {config_name}\n\n" + METATADATA_FILE_FORMAT + process = subprocess.run(["vcfg", "--vhome", vhome, + "update-config-store", "--metadata-file", file_path], + env=os.environ, + cwd=os.environ.get("VOLTTRON_ROOT"), + stderr=subprocess.PIPE, + stdout=subprocess.PIPE + ) + + assert process.stdout.decode('utf-8').strip() == expected_message + assert process.returncode == 1 + + +def test_invalid_config_class(monkeypatch, vhome): + monkeypatch.setenv("VOLTTRON_HOME", vhome) + file_path = os.path.join(vhome, "invalid_config_class.json") + with open(file_path, "w") as f: + f.write(json.dumps({"vip-id-1": "string config"})) + expected_message = f"Metadata for vip-identity vip-id-1 in file {file_path} should be a dictionary or " \ + f"list of dictionary. Got type \n\n" + METATADATA_FILE_FORMAT + process = subprocess.run(["vcfg", "--vhome", vhome, + "update-config-store", "--metadata-file", file_path], + env=os.environ, + cwd=os.environ.get("VOLTTRON_ROOT"), + stderr=subprocess.PIPE, + stdout=subprocess.PIPE + ) + + assert process.stdout.decode('utf-8').strip() == expected_message + assert process.returncode == 1 + + +def test_incorrect_config_type(monkeypatch, vhome): + monkeypatch.setenv("VOLTTRON_HOME", vhome) + file_path = os.path.join(vhome, "invalid_config_type.json") + with open(file_path, "w") as f: + f.write(json.dumps({"vip-id-1": {"config": "string config for json config-type", + "config-type": "json"}})) + expected_message = 'Value for key "config" should be one of the following: \n' \ + '1. filepath \n'\ + '2. string with "config-type" set to "raw" \n'\ + '3. a dictionary \n'\ + '4. list \n\n' + METATADATA_FILE_FORMAT + process = subprocess.run(["vcfg", "--vhome", vhome, + "update-config-store", "--metadata-file", file_path], + env=os.environ, + cwd=os.environ.get("VOLTTRON_ROOT"), + stderr=subprocess.PIPE, + stdout=subprocess.PIPE + ) + + assert process.stdout.decode('utf-8').strip() == expected_message + assert process.returncode == 1 + + +def test_raw_config_in_single_metafile(monkeypatch, vhome): + monkeypatch.setenv("VOLTTRON_HOME", vhome) + file_path = os.path.join(vhome, "single_config.json") + with open(file_path, "w") as f: + f.write(json.dumps({"agent1": {"config": "string config", + "config-type": "raw"}})) + + process = subprocess.run(["vcfg", "--vhome", vhome, + "update-config-store", "--metadata-file", file_path], + env=os.environ, + cwd=os.environ.get("VOLTTRON_ROOT"), + stderr=subprocess.PIPE, + stdout=subprocess.PIPE + ) + + assert process.stdout.decode('utf-8').strip() == '' + assert process.stderr.decode('utf-8').strip() == '' + assert process.returncode == 0 + store_path = os.path.join(vhome, "configuration_store/agent1.store") + assert os.path.isfile(store_path) + with open(store_path) as f: + store = parse_json_config(f.read()) + + assert store["config"] + assert store["config"]["data"] == "string config" + assert store["config"]["type"] == "raw" + initial_modified_time = store["config"]["modified"] + + # now try list of raw config with 1 new and 1 existing + file_path = os.path.join(vhome, "two_config.json") + with open(file_path, "w") as f: + f.write(json.dumps( + {"agent1": [ + {"config": "string config", "config-type": "raw"}, + {"config-name": "new_config", "config-type": "raw", "config": "another string config"} + ] + } + )) + + process = subprocess.run(["vcfg", "--vhome", vhome, + "update-config-store", "--metadata-file", file_path], + env=os.environ, + cwd=os.environ.get("VOLTTRON_ROOT"), + stderr=subprocess.PIPE, + stdout=subprocess.PIPE + ) + + assert process.stdout.decode('utf-8').strip() == '' + assert process.stderr.decode('utf-8').strip() == '' + assert process.returncode == 0 + store_path = os.path.join(vhome, "configuration_store/agent1.store") + assert os.path.isfile(store_path) + with open(store_path) as f: + store = parse_json_config(f.read()) + + assert store["config"] + assert store["config"]["data"] == "string config" + assert store["config"]["type"] == "raw" + assert initial_modified_time == store["config"]["modified"] + + assert store["new_config"] + assert store["new_config"]["data"] == "another string config" + assert store["new_config"]["type"] == "raw" + assert store["new_config"]["modified"] + assert store["new_config"]["modified"] != initial_modified_time + + +def test_json_config_in_single_metafile(monkeypatch, vhome): + monkeypatch.setenv("VOLTTRON_HOME", vhome) + json_data = {"config_key1": "config_value1"} + file_path = os.path.join(vhome, "single_config.json") + with open(file_path, "w") as f: + f.write(json.dumps({"agent2": {"config": json_data, + "config-type": "json"}})) + + process = subprocess.run(["vcfg", "--vhome", vhome, + "update-config-store", "--metadata-file", file_path], + env=os.environ, + cwd=os.environ.get("VOLTTRON_ROOT"), + stderr=subprocess.PIPE, + stdout=subprocess.PIPE + ) + + assert process.stdout.decode('utf-8').strip() == '' + assert process.stderr.decode('utf-8').strip() == '' + assert process.returncode == 0 + store_path = os.path.join(vhome, "configuration_store/agent2.store") + assert os.path.isfile(store_path) + with open(store_path) as f: + store = parse_json_config(f.read()) + + assert store["config"] + assert parse_json_config(store["config"]["data"]) == json_data + assert store["config"]["type"] == "json" + initial_modified_time = store["config"]["modified"] + + +def test_csv_configfile_in_single_metafile(monkeypatch, vhome): + monkeypatch.setenv("VOLTTRON_HOME", vhome) + csv_file = os.path.join(vhome, "config.csv") + csv_str = "point_name,type\npoint1,boolean\npoint2,int" + csv_list = [{"point_name": "point1", "type": "boolean"}, {"point_name": "point2", "type": "int"}] + with open(csv_file, "w") as f: + f.write(csv_str) + file_path = os.path.join(vhome, "single_config.json") + with open(file_path, "w") as f: + f.write(json.dumps({"agent3": {"config": csv_file, + "config-type": "csv"}})) + + process = subprocess.run(["vcfg", "--vhome", vhome, + "update-config-store", "--metadata-file", file_path], + env=os.environ, + cwd=os.environ.get("VOLTTRON_ROOT"), + stderr=subprocess.PIPE, + stdout=subprocess.PIPE + ) + + assert process.stdout.decode('utf-8').strip() == '' + assert process.stderr.decode('utf-8').strip() == '' + assert process.returncode == 0 + store_path = os.path.join(vhome, "configuration_store/agent3.store") + assert os.path.isfile(store_path) + with open(store_path) as f: + store = parse_json_config(f.read()) + + assert store["config"] + f = StringIO(store["config"]["data"]) + csv_list_in_store = [x for x in DictReader(f)] + assert csv_list_in_store == csv_list + assert store["config"]["type"] == "csv" + initial_modified_time = store["config"]["modified"] + + +def test_single_metafile_two_agent(monkeypatch, vhome): + monkeypatch.setenv("VOLTTRON_HOME", vhome) + file_path = os.path.join(vhome, "single_config.json") + with open(file_path, "w") as f: + f.write(json.dumps( + {"agent1": {"config": "string config", "config-type": "raw"}, + "agent2": [ + {"config": "string config", "config-type": "raw"}, + {"config-name": "new_config", "config-type": "raw", "config": "another string config"} + ] + })) + + process = subprocess.run(["vcfg", "--vhome", vhome, + "update-config-store", "--metadata-file", file_path], + env=os.environ, + cwd=os.environ.get("VOLTTRON_ROOT"), + stderr=subprocess.PIPE, + stdout=subprocess.PIPE + ) + + assert process.stdout.decode('utf-8').strip() == '' + assert process.stderr.decode('utf-8').strip() == '' + assert process.returncode == 0 + store_path = os.path.join(vhome, "configuration_store/agent1.store") + assert os.path.isfile(store_path) + with open(store_path) as f: + store = parse_json_config(f.read()) + + assert store["config"] + assert store["config"]["data"] == "string config" + assert store["config"]["type"] == "raw" + assert store["config"]["modified"] + + store_path = os.path.join(vhome, "configuration_store/agent2.store") + assert os.path.isfile(store_path) + with open(store_path) as f: + store = parse_json_config(f.read()) + + assert store["config"] + assert store["config"]["data"] == "string config" + assert store["config"]["type"] == "raw" + assert store["config"]["modified"] + + assert store["new_config"] + assert store["new_config"]["data"] == "another string config" + assert store["new_config"]["type"] == "raw" + assert store["new_config"]["modified"] + + +def test_two_metafile(monkeypatch, vhome): + monkeypatch.setenv("VOLTTRON_HOME", vhome) + file_path1 = os.path.join(vhome, "meta1.json") + with open(file_path1, "w") as f: + f.write(json.dumps({"agent1": {"config": "string config", "config-type": "raw"}})) + file_path2 = os.path.join(vhome, "meta2.json") + with open(file_path2, "w") as f: + f.write(json.dumps( + {"agent2": [ + {"config": "string config", "config-type": "raw"}, + {"config-name": "new_config", "config-type": "raw", "config": "another string config"} + ] + })) + process = subprocess.run(["vcfg", "--vhome", vhome, + "update-config-store", "--metadata-file", file_path1, file_path2], + env=os.environ, + cwd=os.environ.get("VOLTTRON_ROOT"), + stderr=subprocess.PIPE, + stdout=subprocess.PIPE + ) + + assert process.stdout.decode('utf-8').strip() == '' + assert process.stderr.decode('utf-8').strip() == '' + assert process.returncode == 0 + store_path = os.path.join(vhome, "configuration_store/agent1.store") + assert os.path.isfile(store_path) + with open(store_path) as f: + store = parse_json_config(f.read()) + + assert store["config"] + assert store["config"]["data"] == "string config" + assert store["config"]["type"] == "raw" + assert store["config"]["modified"] + + store_path = os.path.join(vhome, "configuration_store/agent2.store") + assert os.path.isfile(store_path) + with open(store_path) as f: + store = parse_json_config(f.read()) + + assert store["config"] + assert store["config"]["data"] == "string config" + assert store["config"]["type"] == "raw" + assert store["config"]["modified"] + + assert store["new_config"] + assert store["new_config"]["data"] == "another string config" + assert store["new_config"]["type"] == "raw" + assert store["new_config"]["modified"] + + +def test_meta_dir(monkeypatch, vhome): + monkeypatch.setenv("VOLTTRON_HOME", vhome) + meta_dir = os.path.join(vhome, "meta_dir") + os.mkdir(meta_dir) + file_path1 = os.path.join(meta_dir, "meta1.json") + with open(file_path1, "w") as f: + f.write(json.dumps({"agent1": {"config": "string config", "config-type": "raw"}})) + file_path2 = os.path.join(meta_dir, "meta2.json") + with open(file_path2, "w") as f: + f.write(json.dumps( + {"agent2": [ + {"config": "string config", "config-type": "raw"}, + {"config-name": "new_config", "config-type": "raw", "config": "another string config"} + ] + })) + process = subprocess.run(["vcfg", "--vhome", vhome, + "update-config-store", "--metadata-file", meta_dir], + env=os.environ, + cwd=os.environ.get("VOLTTRON_ROOT"), + stderr=subprocess.PIPE, + stdout=subprocess.PIPE + ) + + assert process.stdout.decode('utf-8').strip() == '' + assert process.stderr.decode('utf-8').strip() == '' + assert process.returncode == 0 + store_path = os.path.join(vhome, "configuration_store/agent1.store") + assert os.path.isfile(store_path) + with open(store_path) as f: + store = parse_json_config(f.read()) + + assert store["config"] + assert store["config"]["data"] == "string config" + assert store["config"]["type"] == "raw" + assert store["config"]["modified"] + + store_path = os.path.join(vhome, "configuration_store/agent2.store") + assert os.path.isfile(store_path) + with open(store_path) as f: + store = parse_json_config(f.read()) + + assert store["config"] + assert store["config"]["data"] == "string config" + assert store["config"]["type"] == "raw" + assert store["config"]["modified"] + + assert store["new_config"] + assert store["new_config"]["data"] == "another string config" + assert store["new_config"]["type"] == "raw" + assert store["new_config"]["modified"]