diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..0f1deb2 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,26 @@ +# EditorConfig is awesome: http://EditorConfig.org + +# top-most EditorConfig file +root = true + +# Unix-style newlines with a newline ending every file +[*] +end_of_line = lf +insert_final_newline = true +trim_trailing_whitespace = true +charset = utf-8 + +# 4 space indentation +[*.{py,java,r,R}] +indent_style = space +indent_size = 4 + +# 2 space indentation +[*.{js,json,y{a,}ml,html,cwl}] +indent_style = space +indent_size = 2 + +[*.{md,Rmd,rst}] +trim_trailing_whitespace = false +indent_style = space +indent_size = 2 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f739ba8 --- /dev/null +++ b/.gitignore @@ -0,0 +1,23 @@ +*.py[cod] +*.egg-info +*.eggs +.ipynb_checkpoints + +build +dist +.cache +__pycache__ + +htmlcov +.coverage +coverage.xml +.pytest_cache + +docs/_build +docs/apidocs + +# ide +.idea +.eclipse +.vscode +**.swp diff --git a/.prospector.yml b/.prospector.yml new file mode 100644 index 0000000..f790283 --- /dev/null +++ b/.prospector.yml @@ -0,0 +1,29 @@ +# prospector configuration file + +--- + +output-format: grouped + +strictness: medium +doc-warnings: false +test-warnings: true +member-warnings: false + +ignore-paths: + - docs + +pyroma: + run: true + +pep8: + full: true + +pep257: + disable: [ + # Disable because not part of PEP257 official convention: + # see http://pep257.readthedocs.io/en/latest/error_codes.html + D203, # 1 blank line required before class docstring + D212, # Multi-line docstring summary should start at the first line + D213, # Multi-line docstring summary should start at the second line + D404, # First word of the docstring should not be This + ] diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..0e480b1 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,13 @@ +# Config file for automatic testing at travis-ci.org +language: python +python: + - "3.4" + - "3.5" + - "3.6" + +# command to install dependencies +install: + - pip install -r requirements.txt + +# command to run tests +script: python setup.py test diff --git a/CHANGELOG.rst b/CHANGELOG.rst new file mode 100644 index 0000000..f65f005 --- /dev/null +++ b/CHANGELOG.rst @@ -0,0 +1,14 @@ +########### +Change Log +########### + +All notable changes to this project will be documented in this file. +This project adheres to `Semantic Versioning `_. + +[Unreleased] +************ + +Added +----- + +* Empty Python project directory structure diff --git a/CODE_OF_CONDUCT.rst b/CODE_OF_CONDUCT.rst new file mode 100644 index 0000000..af5665c --- /dev/null +++ b/CODE_OF_CONDUCT.rst @@ -0,0 +1,79 @@ +############################################################################### +Contributor Covenant Code of Conduct +############################################################################### + +Our Pledge +********** + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to making participation in our project and +our community a harassment-free experience for everyone, regardless of age, body +size, disability, ethnicity, gender identity and expression, level of experience, +education, socio-economic status, nationality, personal appearance, race, +religion, or sexual identity and orientation. + +Our Standards +************* + +Examples of behavior that contributes to creating a positive environment +include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery and unwelcome sexual attention or + advances +* Trolling, insulting/derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or electronic + address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +Our Responsibilities +******************** + +Project maintainers are responsible for clarifying the standards of acceptable +behavior and are expected to take appropriate and fair corrective action in +response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or +reject comments, commits, code, wiki edits, issues, and other contributions +that are not aligned to this Code of Conduct, or to ban temporarily or +permanently any contributor for other behaviors that they deem inappropriate, +threatening, offensive, or harmful. + +Scope +***** + +This Code of Conduct applies both within project spaces and in public spaces +when an individual is representing the project or its community. Examples of +representing a project or community include using an official project e-mail +address, posting via an official social media account, or acting as an appointed +representative at an online or offline event. Representation of a project may be +further defined and clarified by project maintainers. + +Enforcement +*********** + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported by contacting the project team at l.veen@esciencecenter.nl. All +complaints will be reviewed and investigated and will result in a response that +is deemed necessary and appropriate to the circumstances. The project team is +obligated to maintain confidentiality with regard to the reporter of an incident. +Further details of specific enforcement policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in good +faith may face temporary or permanent repercussions as determined by other +members of the project's leadership. + +Attribution +*********** + +This Code of Conduct is adapted from the `Contributor Covenant `_, version 1.4, +available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst new file mode 100644 index 0000000..2377605 --- /dev/null +++ b/CONTRIBUTING.rst @@ -0,0 +1,44 @@ +############################ +Contributing guidelines +############################ + +We welcome any kind of contribution to our software, from simple comment or question to a full fledged `pull request `_. Please read and follow our `Code of Conduct `_. + +A contribution can be one of the following cases: + +1. you have a question; +1. you think you may have found a bug (including unexpected behavior); +1. you want to make some kind of change to the code base (e.g. to fix a bug, to add a new feature, to update documentation). + +The sections below outline the steps in each case. + +You have a question +******************* + +1. use the search functionality `here `__ to see if someone already filed the same issue; +1. if your issue search did not yield any relevant results, make a new issue; +1. apply the "Question" label; apply other labels when relevant. + +You think you may have found a bug +********************************** + +1. use the search functionality `here `__ to see if someone already filed the same issue; +1. if your issue search did not yield any relevant results, make a new issue, making sure to provide enough information to the rest of the community to understand the cause and context of the problem. Depending on the issue, you may want to include: + - the `SHA hashcode `_ of the commit that is causing your problem; + - some identifying information (name and version number) for dependencies you're using; + - information about the operating system; +1. apply relevant labels to the newly created issue. + +You want to make some kind of change to the code base +***************************************************** + +1. (**important**) announce your plan to the rest of the community *before you start working*. This announcement should be in the form of a (new) issue; +1. (**important**) wait until some kind of consensus is reached about your idea being a good idea; +1. if needed, fork the repository to your own Github profile and create your own feature branch off of the latest master commit. While working on your feature branch, make sure to stay up to date with the master branch by pulling in changes, possibly from the 'upstream' repository (follow the instructions `here `__ and `here `__); +1. make sure the existing tests still work by running ``python setup.py test``; +1. add your own tests (if necessary); +1. update or expand the documentation; +1. `push `_ your feature branch to (your fork of) the Cerise Manager repository on GitHub; +1. create the pull request, e.g. following the instructions `here `__. + +In case you feel like you've made a valuable contribution, but you don't know how to write or run tests for it, or how to generate the documentation: don't let this discourage you from making the pull request; we can help you! Just go ahead and submit the pull request, but keep in mind that you might be asked to append additional commits to your pull request. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..cedf1b6 --- /dev/null +++ b/LICENSE @@ -0,0 +1,204 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..9d5d250 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,2 @@ +include LICENSE +include README.rst diff --git a/NOTICE b/NOTICE new file mode 100644 index 0000000..f20fbd1 --- /dev/null +++ b/NOTICE @@ -0,0 +1,2 @@ +This product includes Cerise Manager, software developed by +Netherlands eScience Center and VU University Amsterdam. diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..d73916f --- /dev/null +++ b/README.rst @@ -0,0 +1,77 @@ +############## +Cerise Manager +############## + +Cerise Manager is a Python library for managing containerised Cerise instances. +Using Cerise Manager, you can create, destroy, start and stop (specialised) +Cerise instances running on your local machine. Cerise Manager is an extension +of Cerise Client, so you can also submit jobs and manage those on the created +services. + +Installation +************ + +Cerise Manager is on PyPI, so you can just + +``` +pip install cerise_manager +``` + +possibly in a virtualenv or Conda environment, if you so desire. Cerise Manager +supports Python 3.4 and up. + +Development +*********** + +To install cerise_manager from GitHub, do: + +.. code-block:: console + + git clone https://github.com/MD-Studio/cerise_manager.git + cd cerise_manager + pip install . + + +Run tests (including coverage) with: + +.. code-block:: console + + python setup.py test + +If you want to contribute to the development of Cerise Manager, +have a look at the `contribution guidelines `_. + +Documentation +------------- + +* Documentation should be put in the ``docs`` folder. The contents have been generated using ``sphinx-quickstart`` (Sphinx version 1.6.5). +* We recommend writing the documentation using Restructured Text (reST) and Google style docstrings. + + - `Restructured Text (reST) and Sphinx CheatSheet `_ + - `Google style docstring examples `_. + +* The documentation is set up with the Read the Docs Sphinx Theme. + + - Check out the `configuration options `_. + +* To generate html documentation run ``python setup.py build_sphinx`` + + - This is configured in ``setup.cfg`` + - Alternatively, run ``make html`` in the ``docs`` folder. + +Legal +***** + +Copyright (c) 2019, Netherlands eScience Center and VU University Amsterdam + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/cerise_manager/__init__.py b/cerise_manager/__init__.py new file mode 100644 index 0000000..f71a595 --- /dev/null +++ b/cerise_manager/__init__.py @@ -0,0 +1,7 @@ +# -*- coding: utf-8 -*- +"""Documentation about Cerise Manager""" + +from .__version__ import __version__ + +__author__ = "Lourens Veen" +__email__ = 'l.veen@esciencecenter.nl' diff --git a/cerise_manager/__version__.py b/cerise_manager/__version__.py new file mode 100644 index 0000000..32bb603 --- /dev/null +++ b/cerise_manager/__version__.py @@ -0,0 +1 @@ +__version__ = '0.0.0.dev' diff --git a/cerise_manager/errors.py b/cerise_manager/errors.py new file mode 100644 index 0000000..4917aad --- /dev/null +++ b/cerise_manager/errors.py @@ -0,0 +1,29 @@ +class ServiceNotFound(Exception): + """ + The given service does not exist on this machine. Either it was + never created, or it was destroyed, or the Docker container running + it was destroyed by an outside influence. + """ + pass + +class ServiceAlreadyExists(Exception): + """ + A service with the given name already exists on this machine, so + another one cannot be created. + """ + pass + +class PortNotAvailable(Exception): + """ + The requested port is occupied by another service, or by some + other program that has bound it on localhost. Either stop the + blocking service or program, or try again with a different + port number. + """ + pass + +class CommunicationError(Exception): + """ + There was an error communicating with the service. + """ + pass diff --git a/cerise_manager/service.py b/cerise_manager/service.py new file mode 100644 index 0000000..67ade9b --- /dev/null +++ b/cerise_manager/service.py @@ -0,0 +1,286 @@ +import errno +import os +import requests +import tarfile +import tempfile +import time + +import docker +import cerise_client.service as ccs + +from cerise_manager import errors + + +# Creating and destroying services + +def create_managed_service(srv_name, port, srv_type, user_name=None, password=None): + """ + Creates a new managed service for a given user at a given port. + + Args: + srv_name (str): A unique name for the service. Must be a valid + Docker container name. + port (int): A unique port number on which the service will be + made available. It will listen only on localhost. + srv_type (str): The type of service to launch. This is the name + of the Docker image to use. + user_name (str): The user name to use to connect to the compute + resource. + password (str): The password to use to connect to the compute + resource. + + Returns: + ManagedService: The created service + + Raises: + ServiceAlreadyExists: A service with this name already exists. + PortNotAvailable: The requested port is occupied. + """ + dc = docker.from_env() + + environment = {} + environment['CERISE_STORE_LOCATION_CLIENT'] = 'http://localhost:{}/files'.format(port) + + if user_name == '': + user_name = None + + if user_name is not None: + environment['CERISE_USERNAME'] = user_name + if password is not None: + environment['CERISE_PASSWORD'] = password + + try: + dc.containers.run( + srv_type, + name=srv_name, + ports={'29593/tcp': ('127.0.0.1', port) }, + environment=environment, + detach=True) + except docker.errors.APIError as e: + # Bit clunky, but it's all Docker gives us... + if 'address already in use' in e.explanation: + raise errors.PortNotAvailable(e) + if 'port is already allocated' in e.explanation: + raise errors.PortNotAvailable(e) + if 'Conflict. The container name' in e.explanation: + raise errors.ServiceAlreadyExists() + raise + + time.sleep(1) + + return ManagedService(srv_name, port) + +def destroy_managed_service(srv): + """ + Destroys a managed service. + + This will make the service unavailable, and delete all + jobs and information about them (including input and output + data) in this service and on the compute resource. + + Args: + srv (ManagedService): A managed service. + + Raises: + ServiceNotFound: A service with this name was not found. + """ + dc = docker.from_env() + try: + container = dc.containers.get(srv._name) + container.stop() + container.remove() + except docker.errors.NotFound: + raise errors.ServiceNotFound() + +def managed_service_exists(srv_name): + """ + Checks whether a managed service with the given name exists. + + Args: + srv_name (str): Name of the service. Must be a valid Docker + container name. + + Returns: + bool: True iff the service exists + """ + dc = docker.from_env() + try: + dc.containers.get(srv_name) + return True + except docker.errors.NotFound: + return False + +def get_managed_service(srv_name, port): + """ + Gets a managed service by name and port. + + Args: + srv_name (str): Name that the service was created with. Must be + a valid Docker container name. + port (int): Port number that the service was created with. + + Returns: + ManagedService: The service, if it exists. + + Raises: + ServiceNotFound: The requested service does not exist. + """ + if not managed_service_exists(srv_name): + raise errors.ServiceNotFound() + return ManagedService(srv_name, port) + +def require_managed_service(srv_name, port, srv_type, user_name=None, password=None): + """ + Creates a new managed service for a given user at a given port, if + it does not already exist. + + If a service with the given name already exists, it is returned + instead and no new service is created. + + Args: + srv_name (str): A unique name for the service. Must be a valid + Docker container name. + port (int): A unique port number on which the service will be + made available. It will listen only on localhost. + srv_type (str): The type of service to launch. This is the name + of the Docker image to use. + user_name (str): The user name to use to connect to the compute + resource. + password (str): The password to use to connect to the compute + resource. + + Returns: + ManagedService: The created service + + Raises: + PortNotAvailable: The requested port is occupied. + """ + try: + return create_managed_service(srv_name, port, srv_type, user_name, password) + except errors.ServiceAlreadyExists: + return get_managed_service(srv_name, port) + + +# Starting and stopping managed services + +def managed_service_is_running(srv): + """ + Checks whether the managed service is running. + + Returns: + bool: True iff the service is running. + """ + dc = docker.from_env() + container = dc.containers.get(srv._name) + return container.status == 'running' + +def start_managed_service(srv): + """ + Start a stopped managed service. + + Does nothing if the service is already running. + """ + dc = docker.from_env() + container = dc.containers.get(srv._name) + container.start() + # Give it some time to start, so subsequent calls work + time.sleep(1) + +def stop_managed_service(srv): + """ + Stop a running managed service. + + This must be done before shutting down the computer, to ensure + a clean shutdown. Does nothing if the service is already + stopped. + """ + dc = docker.from_env() + container = dc.containers.get(srv._name) + container.stop() + + +# Serialisation of services + +def service_to_dict(srv): + """ + Saves the service to a dictionary. + + The dictionary can later be used to recreate the ManagedService + object by passing it to service_from_dict(). The exact format of + the dictionary is not given, but it contains only Python built-in + types so that it can easily be stored or otherwise serialised. + + Returns: + dict: A dictionary with information necessary to rebuild + the ManagedService object. + """ + return { + 'name': srv._name, + 'port': srv._port + } + +def service_from_dict(srv_dict): + """ + Gets a service from a dictionary. + + The dictionary must have been created by a call to + service_to_dict(). + + Args: + srv_dict (dict): A dictionary describing the service. + + Returns: + ManagedService: The service, if it exists. + + Raises: + ServiceNotFound: The requested service does not exist. + """ + return get_managed_service(srv_dict['name'], srv_dict['port']) + + +class ManagedService(ccs.Service): + """A managed service in a Docker container. + """ + def __init__(self, name, port): + """ + Create a new ManagedService object. + + Note that this does not actually create the Docker container; + use create_service(), get_service() or service_from_dict() to + obtain a ManagedService object with an actual corresponding + service. + + Args: + name (str): The name for the service (and its corresponding + Docker container). + port (int): The port number on which the service runs. + """ + super().__init__('http://localhost', port) + + self._name = name + """str: The name of this service, and its Docker container.""" + self._port = port + """int: The port number this service runs on.""" + + def get_log(self): + """ + Get the internal Cerise log for this service. If things are not + working as you expect them to (e.g. a job status of + SystemError), the log may contain useful information on what + went wrong. + + Returns: + str: The job log + """ + dc = docker.from_env() + container = dc.containers.get(self._name) + stream, stat = container.get_archive('/var/log/cerise/cerise_backend.log') + with tempfile.TemporaryFile() as tmp: + tmp.write(stream.read()) + tmp.seek(0) + with tarfile.open(fileobj=tmp) as archive: + # Scope guard does not work in Python 2 + logfile = archive.extractfile('cerise_backend.log') + service_log = logfile.read().decode('utf-8') + logfile.close() + return service_log diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..f53d6b5 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +SPHINXPROJ = cerise_manager +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/_static/theme_overrides.css b/docs/_static/theme_overrides.css new file mode 100644 index 0000000..63ee6cc --- /dev/null +++ b/docs/_static/theme_overrides.css @@ -0,0 +1,13 @@ +/* override table width restrictions */ +@media screen and (min-width: 767px) { + + .wy-table-responsive table td { + /* !important prevents the common CSS stylesheets from overriding + this as on RTD they are loaded after this stylesheet */ + white-space: normal !important; + } + + .wy-table-responsive { + overflow: visible !important; + } +} diff --git a/docs/_templates/.gitignore b/docs/_templates/.gitignore new file mode 100644 index 0000000..e69de29 diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..0fba1c5 --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,207 @@ +# -*- coding: utf-8 -*- +# +# cerise_manager documentation build configuration file, created by +# sphinx-quickstart on Sun Jan 27 11:33:03 2019. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import os +import sys + +import cerise_manager + +here = os.path.dirname(__file__) +sys.path.insert(0, os.path.abspath(os.path.join(here, '..'))) + + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = ['sphinx.ext.autodoc', 'sphinx.ext.napoleon'] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'Cerise Manager' +copyright = u'2019, Netherlands eScience Center and VU University Amsterdam' +author = u"Lourens Veen" + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = cerise_manager.__version__ +# The full version, including alpha/beta/rc tags. +release = cerise_manager.__version__ + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This patterns also effect to html_static_path and html_extra_path +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = False + + +# -- Run apidoc plug-in manually, as readthedocs doesn't support it ------- +# See https://github.com/rtfd/readthedocs.org/issues/1139 +def run_apidoc(_): + here = os.path.dirname(__file__) + out = os.path.abspath(os.path.join(here, 'apidocs')) + src = os.path.abspath(os.path.join(here, '..', 'cerise_manager')) + + ignore_paths = [] + + argv = [ + "-f", + "-T", + "-e", + "-M", + "-o", out, + src + ] + ignore_paths + + try: + # Sphinx 1.7+ + from sphinx.ext import apidoc + apidoc.main(argv) + except ImportError: + # Sphinx 1.6 (and earlier) + from sphinx import apidoc + argv.insert(0, apidoc.__file__) + apidoc.main(argv) + + +def setup(app): + app.connect('builder-inited', run_apidoc) + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'sphinx_rtd_theme' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] +html_context = { + 'css_files': [ + '_static/theme_overrides.css', # override wide tables in RTD theme + ], + } + +# Custom sidebar templates, must be a dictionary that maps document names +# to template names. +# +# This is required for the alabaster theme +# refs: http://alabaster.readthedocs.io/en/latest/installation.html#sidebars +html_sidebars = { + '**': [ + 'relations.html', # needs 'show_related': True theme option to display + 'searchbox.html', + ] +} + + +# -- Options for HTMLHelp output ------------------------------------------ + +# Output file base name for HTML help builder. +htmlhelp_basename = 'cerise_manager_doc' + + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'cerise_manager.tex', u'Cerise Manager Documentation', + u"Lourens Veen", 'manual'), +] + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'cerise_manager', u'Cerise Manager Documentation', + [author], 1) +] + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'cerise_manager', u'Cerise Manager Documentation', + author, 'cerise_manager', "A manager for containerised Cerise instances", + 'Miscellaneous'), +] diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 0000000..38d1256 --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,28 @@ +.. Cerise Manager documentation master file, created by + sphinx-quickstart on Thu Jun 21 11:07:11 2018. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to Cerise Manager's documentation! +========================================================== + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + +API Reference +============= + +.. toctree:: + :maxdepth: 2 + + cerise_manager + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 0000000..145bfc1 --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,36 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build +set SPHINXPROJ=cerise_manager + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% + +:end +popd diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..e69de29 diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..df1c908 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,22 @@ +[metadata] +description-file = README.rst + +[aliases] +# Define `python setup.py test` +test=pytest + +[coverage:run] +#branch = True +source = cerise_manager +omit = tests + +[tool:pytest] +testpaths = tests +addopts = --cov --cov-report xml --cov-report term-missing + +# Define `python setup.py build_sphinx` +[build_sphinx] +source-dir = docs +build-dir = docs/_build +all_files = 1 +builder = html diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..045a3fd --- /dev/null +++ b/setup.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +import os + +from setuptools import setup + +here = os.path.abspath(os.path.dirname(__file__)) + +# To update the package version number, edit cerise_manager/__version__.py +version = {} +with open(os.path.join(here, 'cerise_manager', '__version__.py')) as f: + exec(f.read(), version) + +with open('README.rst') as readme_file: + readme = readme_file.read() + +setup( + name='cerise_manager', + version=version['__version__'], + description="A manager for containerised Cerise instances", + long_description=readme + '\n\n', + author="Lourens Veen", + author_email='l.veen@esciencecenter.nl', + url='https://github.com/MD-Studio/cerise_manager', + packages=[ + 'cerise_manager', + ], + package_dir={'cerise_manager': + 'cerise_manager'}, + include_package_data=True, + license="Apache Software License 2.0", + zip_safe=False, + keywords='cerise_manager', + classifiers=[ + 'Development Status :: 2 - Pre-Alpha', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: Apache Software License', + 'Natural Language :: English', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + ], + test_suite='tests', + install_requires=[ + 'cerise_client', + 'defusedxml==0.5.0', + 'docker<3', + 'future', + 'idna<2.8' + ], # FIXME: add your package's dependencies to this list + setup_requires=[ + # dependency for `python setup.py test` + 'pytest-runner', + # dependencies for `python setup.py build_sphinx` + 'sphinx', + 'sphinx_rtd_theme', + 'recommonmark' + ], + tests_require=[ + 'coverage<5', + 'pytest', + 'pytest-cov', + 'pycodestyle', + ], + extras_require={ + 'dev': ['prospector[with_pyroma]', 'yapf', 'isort'], + } +) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..40a96af --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1 @@ +# -*- coding: utf-8 -*- diff --git a/tests/clean_up.py b/tests/clean_up.py new file mode 100644 index 0000000..e9023ac --- /dev/null +++ b/tests/clean_up.py @@ -0,0 +1,17 @@ +import docker + +def clean_up_service(srv_name): + dc = docker.from_env() + try: + test_srv = dc.containers.get(srv_name) + test_srv.stop() + test_srv.remove() + except docker.errors.NotFound: + pass + +# Clean up any mess left over from previous failed tests. +def clean_up(): + clean_up_service('cerise_manager_test_service') + clean_up_service('cerise_manager_test_service2') + + diff --git a/tests/fixtures.py b/tests/fixtures.py new file mode 100644 index 0000000..8d212ed --- /dev/null +++ b/tests/fixtures.py @@ -0,0 +1,59 @@ +import docker +import os +import pytest +import requests +import time + +import cerise_manager.service as cs + +@pytest.fixture() +def docker_client(request): + return docker.from_env() + +@pytest.fixture(scope="module") +def test_image(request): + """Get a plain cerise image for testing. + + Ignores errors; we may have a local image available already, + in which case we want to continue, otherwise the other + tests will fail. + """ + docker_client = docker.from_env() + try: + docker_client.images.pull('mdstudio/cerise:develop') + except docker.errors.APIError: + pass + return 'mdstudio/cerise:develop' + +@pytest.fixture() +def test_service(request, test_image): + from .clean_up import clean_up_service + clean_up_service('cerise_manager_test_service') + srv = cs.create_managed_service('cerise_manager_test_service', 29593, + test_image) + + yield srv + + clean_up_service('cerise_manager_test_service') + +@pytest.fixture() +def this_dir(request): + return os.path.dirname(__file__) + +def create_test_job(test_service, this_dir, name): + job = test_service.create_job(name) + r = requests.get('http://localhost:29593/files/input/' + name + '/') + assert r.status_code == 200 + + # run job to create some outputs + job.set_workflow(os.path.join(this_dir, 'test_workflow2.cwl')) + job.add_input_file('input_file', os.path.join(this_dir, 'test_workflow2.cwl')) + job.run() + while job.is_running(): + time.sleep(0.1) + assert job.state == 'Success' + + counts = job.outputs['counts'] + assert counts.text != '' + return job + diff --git a/tests/lint.py b/tests/lint.py new file mode 100644 index 0000000..df4d6b9 --- /dev/null +++ b/tests/lint.py @@ -0,0 +1,43 @@ +""" Lint tests """ +import os +import textwrap + +import pycodestyle # formerly known as pep8 + + +def test_pep8_conformance(): + """Test that we conform to PEP-8.""" + check_paths = [ + 'cerise_manager', + 'tests', + ] + exclude_paths = [] + + print("PEP8 check of directories: {}\n".format(', '.join(check_paths))) + + # Get paths wrt package root + package_root = os.path.dirname(os.path.dirname(__file__)) + for paths in (check_paths, exclude_paths): + for i, path in enumerate(paths): + paths[i] = os.path.join(package_root, path) + + style = pycodestyle.StyleGuide() + style.options.exclude.extend(exclude_paths) + + success = style.check_files(check_paths).total_errors == 0 + + if not success: + print(textwrap.dedent(""" + Your Python code does not conform to the official Python style + guide (PEP8), see https://www.python.org/dev/peps/pep-0008 + + A list of warning and error messages can be found above, + prefixed with filename:line number:column number. + + Run `yapf -i yourfile.py` to automatically fix most errors. + Run `yapf -d yourfile.py` to preview what would be changed. + Run `pip install --upgrade yapf` to install the latest version + of yapf. + """)) + + assert success, "Your code does not conform to PEP8" diff --git a/tests/test_service.py b/tests/test_service.py new file mode 100644 index 0000000..386e226 --- /dev/null +++ b/tests/test_service.py @@ -0,0 +1,204 @@ +import docker +import json +import os +import pytest +import requests +import sys +import time + + +# clean up any mess left over from previous failed tests +from .clean_up import clean_up +clean_up() + +import cerise_manager.service as cs +import cerise_manager.errors as ce +from .clean_up import clean_up_service + +from .fixtures import docker_client, test_image, test_service, this_dir +from .fixtures import create_test_job + +@pytest.fixture() +def test_container(request, test_image, docker_client): + try: + container = docker_client.containers.get('cerise_manager_test_service') + except docker.errors.NotFound: + image = docker_client.images.get(test_image) + + container = docker_client.containers.run( + image, + name='cerise_manager_test_service', + ports={'29593/tcp': ('127.0.0.1', 29593) }, + detach=True) + + yield container + + container.stop() + container.remove() + +@pytest.fixture() +def test_service_dict(request): + return { + 'name': 'cerise_manager_test_service', + 'port': 29593 + } + +def test_managed_service_exists(test_container): + exists = cs.managed_service_exists('cerise_manager_test_service') + assert isinstance(exists, bool) + assert exists + +def test_managed_service_does_not_exist(): + exists = cs.managed_service_exists('cerise_manager_test_service') + assert isinstance(exists, bool) + assert not exists + + +def test_get_managed_service(test_container): + srv = cs.get_managed_service('cerise_manager_test_service', 29593) + assert isinstance(srv, cs.ManagedService) + +def test_get_missing_managed_service(): + with pytest.raises(ce.ServiceNotFound): + cs.get_managed_service('cerise_manager_test_service', 29593) + +def test_service_from_dict(test_container, test_service_dict): + srv = cs.service_from_dict(test_service_dict) + assert isinstance(srv, cs.ManagedService) + +def test_missing_service_from_dict(test_service_dict): + with pytest.raises(ce.ServiceNotFound): + cs.service_from_dict(test_service_dict) + +def test_create_managed_service(docker_client): + srv = cs.create_managed_service('cerise_manager_test_service', 29593, + 'mdstudio/cerise:develop') + assert isinstance(srv, cs.ManagedService) + clean_up_service('cerise_manager_test_service') + +def test_create_existing_managed_service(test_container): + with pytest.raises(ce.ServiceAlreadyExists): + cs.create_managed_service('cerise_manager_test_service', 29593, + 'mdstudio/cerise:develop') + +def test_create_managed_service_port_occupied(test_container): + with pytest.raises(ce.PortNotAvailable): + cs.create_managed_service('cerise_manager_test_service2', 29593, + 'mdstudio/cerise:develop') + clean_up_service('cerise_manager_test_service2') + +def test_create_two_services(test_container): + srv = cs.create_managed_service('cerise_manager_test_service2', 29594, + 'mdstudio/cerise:develop') + assert isinstance(srv, cs.ManagedService) + cs.destroy_managed_service(srv) + +def test_create_managed_service_object(): + srv = cs.ManagedService('cerise_manager_test_service', 29593) + assert srv._name == 'cerise_manager_test_service' + assert srv._port == 29593 + +def test_destroy_managed_service(docker_client, test_service): + container = docker_client.containers.get('cerise_manager_test_service') + assert container.status == 'running' + + cs.destroy_managed_service(test_service) + + with pytest.raises(docker.errors.NotFound): + docker_client.containers.get('cerise_manager_test_service') + +def test_destroy_missing_managed_service(docker_client): + srv = cs.ManagedService('non_existing_service', 29593) + with pytest.raises(ce.ServiceNotFound): + cs.destroy_managed_service(srv) + +def test_require_managed_service(docker_client): + srv = cs.require_managed_service('cerise_manager_test_service', 29593, + 'mdstudio/cerise:develop') + assert isinstance(srv, cs.ManagedService) + clean_up_service('cerise_manager_test_service') + +def test_require_existing_managed_service(docker_client, test_service): + srv = cs.require_managed_service('cerise_manager_test_service', 29593, + 'mdstudio/cerise:develop') + assert isinstance(srv, cs.ManagedService) + assert srv._name == 'cerise_manager_test_service' + assert srv._port == 29593 + +def test_require_managed_server_occupied_port(docker_client, test_service): + with pytest.raises(ce.PortNotAvailable): + srv = cs.require_managed_service('cerise_manager_test_service2', 29593, + 'mdstudio/cerise:develop') + +def test_start_running_service(docker_client, test_service): + container = docker_client.containers.get('cerise_manager_test_service') + assert container.status == 'running' + + cs.start_managed_service(test_service) + + container.reload() + assert container.status == 'running' + +def test_start_stopped_service(docker_client, test_service): + container = docker_client.containers.get('cerise_manager_test_service') + assert container.status == 'running' + container.stop() + container.reload() + assert container.status == 'exited' + + cs.start_managed_service(test_service) + + container.reload() + assert container.status == 'running' + +def test_stop_running_managed_service(docker_client, test_service): + cs.stop_managed_service(test_service) + + container = docker_client.containers.get('cerise_manager_test_service') + assert container.status == 'exited' + +def test_stop_stopped_managed_service(docker_client, test_service): + container = docker_client.containers.get('cerise_manager_test_service') + container.stop() + container.reload() + assert container.status == 'exited' + + cs.stop_managed_service(test_service) + + container.reload() + assert container.status == 'exited' + +def test_managed_service_is_running(docker_client, test_service): + container = docker_client.containers.get('cerise_manager_test_service') + assert container.status == 'running' + assert cs.managed_service_is_running(test_service) + +def test_is_not_running(docker_client, test_service): + container = docker_client.containers.get('cerise_manager_test_service') + container.stop() + container.reload() + assert container.status == 'exited' + + assert not cs.managed_service_is_running(test_service) + +def test_service_to_dict(test_service): + dict_ = cs.service_to_dict(test_service) + assert dict_['name'] == 'cerise_manager_test_service' + assert dict_['port'] == 29593 + +def test_service_serialisation(test_service): + dict_ = cs.service_to_dict(test_service) + json_dict = json.dumps(dict_) + dict2 = json.loads(json_dict) + srv = cs.service_from_dict(dict2) + + assert cs.managed_service_is_running(srv) + assert srv._name == 'cerise_manager_test_service' + assert srv._port == 29593 + +def test_get_log(test_service): + # Give it a bit of time to start up, esp. on Travis + time.sleep(5) + log = test_service.get_log() + assert isinstance(log, str) or isinstance(log, unicode) + assert log != ''