Merge branch 'release-0.2.3'

CHANGELOG.rst

@@ -5,6 +5,15 @@ Change Log
 All notable changes to this project will be documented in this file.
 This project adheres to `Semantic Versioning <http://semver.org/>`_.
 
+0.2.3
+*****
+
+Fixed
+-----
+
+* Don't litter slurm-xyz.out files all over the place
+* Update Docker to Ubuntu 18.04
+
 0.2.2
 *****
 

Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:16.04
+FROM ubuntu:18.04
 
 # Install requirements
 RUN apt-get update -y && apt-get -y dist-upgrade && \

README.md

@@ -1,6 +1,6 @@
 Cerise
 ======
-[![Master build status](https://api.travis-ci.org/MD-Studio/cerise.svg?branch=master)](https://travis-ci.org/MD-Studio/cerise) [![Codacy Badge](https://api.codacy.com/project/badge/Grade/56de5791221a42e5964ba9d3a949c9c4)](https://www.codacy.com/app/LourensVeen/cerise) [![Coverage Badge](https://api.codacy.com/project/badge/Coverage/56de5791221a42e5964ba9d3a949c9c4)](https://www.codacy.com/app/LourensVeen/cerise) [![Documentation Status](https://readthedocs.org/projects/cerise/badge/?version=stable)](http://cerise.readthedocs.io/en/latest/?badge=latest) [![Docker Build Status](https://img.shields.io/docker/build/mdstudio/cerise.svg)](https://hub.docker.com/r/mdstudio/cerise/)
+[![Master build status](https://api.travis-ci.org/MD-Studio/cerise.svg?branch=master)](https://travis-ci.org/MD-Studio/cerise) [![Codacy Badge](https://api.codacy.com/project/badge/Grade/56de5791221a42e5964ba9d3a949c9c4)](https://www.codacy.com/app/LourensVeen/cerise) [![Coverage Badge](https://api.codacy.com/project/badge/Coverage/56de5791221a42e5964ba9d3a949c9c4)](https://www.codacy.com/app/LourensVeen/cerise) [![Documentation Status](https://readthedocs.org/projects/cerise/badge/?version=stable)](http://cerise.readthedocs.io/en/latest/?badge=stable) [![Docker Build Status](https://img.shields.io/docker/build/mdstudio/cerise.svg)](https://hub.docker.com/r/mdstudio/cerise/)
 
 This is a simple REST service that can run (some) CWL jobs on some remote
 compute resource. It uses a REST API as its interface and Cerulean

cerise/back_end/job_runner.py

@@ -83,6 +83,8 @@ def start_job(self, job_id: str) -> None:
             ]
             jobdesc.stdout_file = job.remote_stdout_path
             jobdesc.stderr_file = job.remote_stderr_path
+            jobdesc.system_out_file = job.remote_system_out_path
+            jobdesc.system_err_file = job.remote_system_err_path
 
             if job.time_limit > 0:
                 jobdesc.time_reserved = job.time_limit

cerise/back_end/remote_job_files.py

@@ -100,6 +100,10 @@ def stage_job(self, job_id: str, input_files: List[File],
             # configure output
             job.remote_stdout_path = str(self._abs_path(job_id, 'stdout.txt'))
             job.remote_stderr_path = str(self._abs_path(job_id, 'stderr.txt'))
+            job.remote_system_out_path = str(self._abs_path(job_id,
+                                                            'sysout.txt'))
+            job.remote_system_err_path = str(self._abs_path(job_id,
+                                                            'syserr.txt'))
 
     def destage_job_output(self, job_id: str) -> List[File]:
         """Download results of the given job from the compute resource.

cerise/back_end/test/conftest.py

@@ -231,6 +231,8 @@ def mock_store_staged(request, mock_config):
     job.remote_input_path = str(job_dir / 'input.json')
     job.remote_stdout_path = str(job_dir / 'stdout.txt')
     job.remote_stderr_path = str(job_dir / 'stderr.txt')
+    job.remote_system_out_path = str(job_dir / 'sysout.txt')
+    job.remote_system_err_path = str(job_dir / 'syserr.txt')
     job.state = JobState.STAGING_IN
 
     store.add_job(job)

cerise/back_end/test/mock_job.py

@@ -70,6 +70,10 @@ def __init__(self, job_id, name, workflow, job_input):
         """str: The absolute remote path of the standard output dump."""
         self.remote_stderr_path = ''
         """str: The absolute remote path of the standard error dump."""
+        self.remote_system_out_path = ''
+        """str: The absolute remote path of the system output dump."""
+        self.remote_system_err_path = ''
+        """str: The absolute remote path of the standard error dump."""
 
         # Post-destaging data
         self.local_output = ''

cerise/job_store/sqlite_job.py

@@ -202,6 +202,26 @@ def remote_stderr_path(self) -> str:
     def remote_stderr_path(self, value: str) -> None:
         self._set_var('remote_stderr_path', value)
 
+    @property
+    def remote_system_out_path(self) -> str:
+        """The absolute remote path of the system out dump.
+        """
+        return cast(str, self._get_var('remote_system_out_path'))
+
+    @remote_system_out_path.setter
+    def remote_system_out_path(self, value: str) -> None:
+        self._set_var('remote_system_out_path', value)
+
+    @property
+    def remote_system_err_path(self) -> str:
+        """The absolute remote path of the system error dump.
+        """
+        return cast(str, self._get_var('remote_system_err_path'))
+
+    @remote_system_err_path.setter
+    def remote_system_err_path(self, value: str) -> None:
+        self._set_var('remote_system_err_path', value)
+
     # Post-destaging data
     @property
     def local_output(self) -> str:

cerise/job_store/sqlite_job_store.py

@@ -64,6 +64,8 @@ def __init__(self, dbfile: str) -> None:
                 remote_input_path VARCHAR(255) DEFAULT '',
                 remote_stdout_path VARCHAR(255) DEFAULT '',
                 remote_stderr_path VARCHAR(255) DEFAULT '',
+                remote_system_out_path VARCHAR(255) DEFAULT '',
+                remote_system_err_path VARCHAR(255) DEFAULT '',
                 remote_job_id VARCHAR(255),
                 local_output TEXT DEFAULT ''
                 )

cerise/job_store/test/test_sqlite_job_store.py

@@ -35,6 +35,8 @@ def inited_db(request, empty_db):
                              '    remote_input_path VARCHAR(255),'
                              '    remote_stdout_path VARCHAR(255),'
                              '    remote_stderr_path VARCHAR(255),'
+                             '    remote_system_out_path VARCHAR(255),'
+                             '    remote_system_err_path VARCHAR(255),'
                              '    remote_job_id VARCHAR(255),'
                              '    local_output TEXT'
                              '    )')
@@ -247,6 +249,16 @@ def test_set_get_remote_stderr_path(job):
     assert job.remote_stderr_path == '/test_set_get'
 
 
+def test_set_get_remote_system_out_path(job):
+    job.remote_system_out_path = '/test_set_get_out'
+    assert job.remote_system_out_path == '/test_set_get_out'
+
+
+def test_set_get_remote_system_err_path(job):
+    job.remote_system_err_path = '/test_set_get_err'
+    assert job.remote_system_err_path == '/test_set_get_err'
+
+
 def test_set_get_local_output(job):
     local_output = '{Local output\n Not really}'
     job.local_output = local_output

cerise/test/test_service.py

@@ -186,6 +186,23 @@ def debug_output(request, tmpdir, cerise_client):
         f.write(stream.read())
 
 
+@pytest.fixture
+def debug_print_log(tmpdir, cerise_client):
+    yield
+
+    # print logs for debugging
+    client = docker.from_env()
+    service = client.containers.get('cerise-test-service')
+    archive_file = tmpdir / 'docker_logs2.tar'
+    stream, _ = service.get_archive('/var/log')
+    with archive_file.open('wb') as f:
+        f.write(stream.read())
+
+    with tarfile.open(str(archive_file)) as tf:
+        with tf.extractfile('log/cerise/cerise_backend.log') as f:
+            print(f.read().decode('utf-8'))
+
+
 def _start_job(cerise_client, webdav_client, job_fixture, test_name=None):
     if test_name is None:
         test_name = 'test_post_' + job_fixture.__name__
@@ -428,7 +445,7 @@ def test_restart_service(cerise_service, cerise_client, webdav_client,
 
 
 def test_dropped_ssh_connection(cerise_service, cerise_client, webdav_client,
-                                slurm_container):
+                                slurm_container, debug_print_log):
     job = _start_job(cerise_client, webdav_client, SlowJob,
                      'test_dropped_ssh_connection')
     _drop_connections(slurm_container)
@@ -438,7 +455,7 @@ def test_dropped_ssh_connection(cerise_service, cerise_client, webdav_client,
 
 
 def test_no_resource_connection(cerise_service, cerise_client, webdav_client,
-                                slurm_container):
+                                slurm_container, debug_print_log):
     _drop_network(slurm_container)
     time.sleep(1)
     job = _start_job(cerise_client, webdav_client, LongRunningJob,
@@ -455,6 +472,7 @@ def test_no_resource_connection(cerise_service, cerise_client, webdav_client,
 
     job, response = cerise_client.jobs.get_job_by_id(jobId=job.id).result()
     assert response.status_code == 200
+
     assert job.state == 'Running'
 
     time.sleep(5)

docs/source/conf.py

@@ -57,9 +57,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = '0.2.2'
+version = '0.2.3'
 # The full version, including alpha/beta/rc tags.
-release = '0.2.2'
+release = '0.2.3'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

requirements.txt

@@ -1,4 +1,4 @@
-cerulean >= 0.3.5
+cerulean >= 0.3.6
 connexion == 1.0.129
 gevent >= 0.13
 gunicorn >= 19.7.1

setup.py

@@ -8,7 +8,7 @@ def read(fname):
 
 setup(
     name = "cerise",
-    version = "0.2.2",
+    version = "0.2.3",
     author = "Lourens Veen",
     author_email = "[email protected]",
     description = ("A simple CWL job running service"),