Merge pull request #15 from GRIDAPPSD/releases/2020.08.0

Release of version 2020.08.0

.gitignore

@@ -0,0 +1,6 @@
+env/
+.idea/
+__pycache__/
+.pytest_cache/
+dumps/
+venv/

.travis.yml

@@ -0,0 +1,31 @@
+sudo: 'required'
+
+language: python
+
+python:
+  - 3.6.10
+
+addons:
+  artifacts:
+    s3_region: "us-west-2"
+    paths:
+      - out
+    target_paths:
+      - /$TRAVIS_REPO_SLUG/
+
+services:
+  - docker
+
+install:
+  - pip3 install -r requirements.txt 
+
+jobs:
+  include:
+  - stage: test
+    name: "Integration Tests"
+    script: 
+      - mkdir out
+      - pytest -rA --html=out/gridappsd-testing_$(date +"%Y%m%d_%H%M%S")_${TRAVIS_BRANCH}.html --self-contained-html
+
+stages:
+  - test

README

@@ -1,17 +1,15 @@
 
 Python script to start the gridappsd platform for automated testing
 
-pip install docker
-
-On OSX to fix cert error:
-/Applications/Python\ 3.8//Install\ Certificates.command
-
-
-todo:
-  - try: dictionary items
+*** This will stop all your running containers ***
 
+*** Requires python 3.6 ***
 
-*** This will stop all your running containers ***
+# setup python environment
+sudo apt install python3.6 python3.6-venv
+python3.6 -m venv env
+source env/bin/activate
+pip3 install -r requirements.txt 
 
-to start the containers
-  python3 gridapspd_testing.py
+# run pytest
+pytest -s

conftest.py

@@ -0,0 +1,106 @@
+from copy import deepcopy
+from pathlib import Path
+
+import pytest
+from gridappsd import GOSS, GridAPPSD
+from gridappsd.docker_handler import (run_dependency_containers, run_gridappsd_container, Containers,
+                                      run_containers, DEFAULT_GRIDAPPSD_DOCKER_CONFIG)
+
+# Assumes tests directory is within the directory that should be mounted to the
+# gridappsd container
+LOCAL_MOUNT_POINT_FOR_SERVICE = Path(__file__).parent.parent.absolute()
+
+# Mount point inside the gridappsd container itself.
+SERVICE_MOUNT_POINT = "/gridappsd/services/gridappsd-sensor-simulator"
+CONFIG_MOUNT_POINT = "/gridappsd/services/sensor_simulator.config"
+
+# If set to False then None of the containers will clean up after themselves.
+# If more than one test is ran then this will cause an error because the gridappsd
+# container will not be cleansed.
+STOP_AFTER_FIXTURE = True
+
+
+@pytest.fixture(scope="module")
+def docker_dependencies():
+    print("Docker dependencies")
+    Containers.reset_all_containers()
+
+    with run_dependency_containers(stop_after=STOP_AFTER_FIXTURE) as dep:
+        yield dep
+    print("Cleanup docker dependencies")
+
+
+@pytest.fixture
+def goss_client(docker_dependencies):
+    with run_gridappsd_container(STOP_AFTER_FIXTURE):
+        goss = GOSS()
+        goss.connect()
+        assert goss.connected
+
+        yield goss
+
+        goss.disconnect()
+
+
+@pytest.fixture
+def gridappsd_client(docker_dependencies):
+    with run_gridappsd_container(True):
+        gappsd = GridAPPSD()
+        gappsd.connect()
+        assert gappsd.connected
+
+        yield gappsd
+
+        gappsd.disconnect()
+
+# USED AS EXAMPLES COPIED FROM gridappsd-sensor-simulator
+#
+# @pytest.fixture(scope="module")
+# def gridappsd_client_include_as_service_no_cleanup(docker_dependencies):
+#
+#     config = deepcopy(DEFAULT_GRIDAPPSD_DOCKER_CONFIG)
+#
+#     config['gridappsd']['volumes'][str(LOCAL_MOUNT_POINT_FOR_SERVICE)] = dict(
+#         bind=str(SERVICE_MOUNT_POINT),
+#         mode="rw")
+#
+#     # from pprint import pprint
+#     # pprint(config['gridappsd'])
+#     with run_containers(config, stop_after=False) as containers:
+#         containers.wait_for_log_pattern("gridappsd", "MYSQL")
+#
+#         gappsd = GridAPPSD()
+#         gappsd.connect()
+#         assert gappsd.connected
+#
+#         yield gappsd
+#
+#         gappsd.disconnect()
+#
+#
+# @pytest.fixture
+# def gridappsd_client_include_as_service(docker_dependencies):
+#
+#     config = deepcopy(DEFAULT_GRIDAPPSD_DOCKER_CONFIG)
+#
+#     config['gridappsd']['volumes'][str(LOCAL_MOUNT_POINT_FOR_SERVICE)] = dict(
+#         bind=str(SERVICE_MOUNT_POINT),
+#         mode="rw")
+#
+#     local_config = LOCAL_MOUNT_POINT_FOR_SERVICE.joinpath("sensor_simulator.config")
+#     config['gridappsd']['volumes'][str(local_config)] = dict(
+#         bind=str(CONFIG_MOUNT_POINT),
+#         mode="rw")
+#
+#     # from pprint import pprint
+#     # pprint(config['gridappsd'])
+#     with run_containers(config, stop_after=STOP_AFTER_FIXTURE) as containers:
+#         containers.wait_for_log_pattern("gridappsd", "MYSQL")
+#
+#         gappsd = GridAPPSD()
+#         gappsd.connect()
+#         assert gappsd.connected
+#
+#         yield gappsd
+#
+#         gappsd.disconnect()

gridappsd_docker.py

@@ -0,0 +1,195 @@
+#!/usr/bin/env python3
+
+import docker
+import os
+import re
+import shutil
+import time
+import urllib.request
+
+
+def expand_all(user_path):
+    return os.path.expandvars(os.path.expanduser(user_path))
+
+
+# This path needs to be the path to the repo where the data is done.
+GRIDAPPSD_TEST_REPO = expand_all(os.environ.get("GRIDAPPSD_TEST_REPO", os.path.dirname(__file__)))
+GRIDAPPSD_DATA_REPO = expand_all(os.environ.get("GRIDAPPSD_DATA_REPO", "/tmp/gridappsd_temp_data"))
+
+if not os.path.exists(GRIDAPPSD_TEST_REPO):
+    raise AttributeError(f"Invalid GRIDAPPSD_TEST_REPO {GRIDAPPSD_TEST_REPO}")
+
+os.makedirs(GRIDAPPSD_DATA_REPO, exist_ok=True)
+
+repo_dir = GRIDAPPSD_TEST_REPO
+data_dir = GRIDAPPSD_DATA_REPO
+
+
+gridappsd_docker_config = {
+    'influxdb': {
+        'start': True,
+        'image': 'gridappsd/influxdb:develop',
+        'pull': True,
+        'ports': {'8086/tcp': 8086},
+        'environment': {"INFLUXDB_DB": "proven"},
+        'links': '',
+        'volumes': '',
+        'entrypoint': '',
+    }
+    ,
+    'redis': {
+        'start': True,
+        'image': 'redis:3.2.11-alpine',
+        'pull': True,
+        'ports': {'6379/tcp': 6379},
+        'environment': [],
+        'links': '',
+        'volumes': '',
+        'entrypoint': 'redis-server --appendonly yes',
+    },
+    'blazegraph': {
+        'start': True,
+        'image': 'gridappsd/blazegraph:develop',
+        'pull': True,
+        'ports': {'8080/tcp': 8889},
+        'environment': [],
+        'links': '',
+        'volumes': '',
+        'entrypoint': '',
+    },
+    'mysql': {
+        'start': True,
+        'image': 'mysql/mysql-server:5.7',
+        'pull': True,
+        'ports': {'3306/tcp': 3306},
+        'environment': {
+            "MYSQL_RANDOM_ROOT_PASSWORD": "yes",
+            "MYSQL_PORT": '3306'
+        },
+        'links': '',
+        'volumes': {
+            data_dir + '/dumps/gridappsd_mysql_dump.sql': {'bind': '/docker-entrypoint-initdb.d/schema.sql',
+                                                           'mode': 'ro'}
+        },
+        'entrypoint': '',
+    },
+    'proven': {
+        'start': True,
+        'image': 'gridappsd/proven:develop',
+        'pull': True,
+        'ports': {'8080/tcp': 18080},
+        'environment': {
+            "PROVEN_SERVICES_PORT": "18080",
+            "PROVEN_SWAGGER_HOST_PORT": "localhost:18080",
+            "PROVEN_USE_IDB": "true",
+            "PROVEN_IDB_URL": "http://influxdb:8086",
+            "PROVEN_IDB_DB": "proven",
+            "PROVEN_IDB_RP": "autogen",
+            "PROVEN_IDB_USERNAME": "root",
+            "PROVEN_IDB_PASSWORD": "root",
+            "PROVEN_T3DIR": "/proven"},
+        'links': {'influxdb': 'influxdb'},
+        'volumes': '',
+        'entrypoint': '',
+    },
+    'gridappsd': {
+        'start': True,
+        'image': 'gridappsd/gridappsd:develop',
+        'pull': True,
+        'ports': {'61613/tcp': 61613, '61614/tcp': 61614, '61616/tcp': 61616},
+        'environment': {
+            "PATH": "/gridappsd/bin:/gridappsd/lib:/gridappsd/services/fncsgossbridge/service:/usr/local/bin:/usr/local/sbin:/usr/sbin:/usr/bin:/sbin:/bin",
+            "DEBUG": 1,
+            "START": 1
+        },
+        'links': {'mysql': 'mysql', 'influxdb': 'influxdb', 'blazegraph': 'blazegraph', 'proven': 'proven',
+                  'redis': 'redis'},
+        'volumes': {
+            repo_dir + '/conf/entrypoint.sh': {'bind': '/gridappsd/entrypoint.sh', 'mode': 'rw'},
+            repo_dir + '/conf/run-gridappsd.sh': {'bind': '/gridappsd/run-gridappsd.sh', 'mode': 'rw'}
+        },
+        'entrypoint': '',
+    }
+}
+
+
+def docker_down(client=None):
+    if client is None:
+        client = docker.from_env()
+
+    # Stop all containers
+    print("\nStopping all containers")
+    for container in client.containers.list():
+        container.stop()
+        time.sleep(5)
+
+    print("\nRemoving previous data")
+    path = '{}/gridappsd'.format(data_dir)
+    if os.path.isdir(path):
+        shutil.rmtree(path, ignore_errors=False, onerror=None)
+
+
+def docker_up(docker_config=None):
+    global gridappsd_docker_config
+    if docker_config is not None:
+        gridappsd_docker_config = docker_config
+
+    client = docker.from_env()
+
+    # Start from scratch
+    docker_down(client)
+
+    # Downlaod mysql file
+    print("\nDownloading mysql file")
+    mysql_dir = '{}/dumps'.format(data_dir)
+    mysql_file = '{}/gridappsd_mysql_dump.sql'.format(mysql_dir)
+    if not os.path.isdir(mysql_dir):
+        os.makedirs(mysql_dir, 0o0775)
+    urllib.request.urlretrieve('https://raw.githubusercontent.com/GRIDAPPSD/Bootstrap/master/gridappsd_mysql_dump.sql',
+                               filename=mysql_file)
+
+    # Modify the mysql file to allow connections from gridappsd container
+    with open(mysql_file, "r") as sources:
+        lines = sources.readlines()
+    with open(mysql_file, "w") as sources:
+        for line in lines:
+            sources.write(re.sub(r'localhost', '%', line))
+
+    # Pull the container
+    print ("\n")
+    for service, value in gridappsd_docker_config.items():
+     if gridappsd_docker_config[service]['pull']:
+       print ("Pulling %s : %s" % ( service, gridappsd_docker_config[service]['image']))
+       client.images.pull(gridappsd_docker_config[service]['image'])
+
+    # Start the container
+    print("\n")
+    for service, value in gridappsd_docker_config.items():
+        if gridappsd_docker_config[service]['start']:
+            print("Starting %s : %s" % (service, gridappsd_docker_config[service]['image']))
+            kwargs = {}
+            kwargs['image'] = gridappsd_docker_config[service]['image']
+            # Only name the containers if remove is on
+            kwargs['remove'] = True
+            kwargs['name'] = service
+            kwargs['detach'] = True
+            if gridappsd_docker_config[service]['environment']:
+                kwargs['environment'] = gridappsd_docker_config[service]['environment']
+            if gridappsd_docker_config[service]['ports']:
+                kwargs['ports'] = gridappsd_docker_config[service]['ports']
+            if gridappsd_docker_config[service]['volumes']:
+                kwargs['volumes'] = gridappsd_docker_config[service]['volumes']
+            if gridappsd_docker_config[service]['entrypoint']:
+                kwargs['entrypoint'] = gridappsd_docker_config[service]['entrypoint']
+            if gridappsd_docker_config[service]['links']:
+                kwargs['links'] = gridappsd_docker_config[service]['links']
+            # print (kwargs)
+            container = client.containers.run(**kwargs)
+            gridappsd_docker_config[service]['containerid'] = container.id
+
+    time.sleep(60)
+
+    # List all running containers
+    print("\n\nList all containers")
+    for container in client.containers.list():
+        print(container.name)