Logs refactorization and containerization (#329)

.gitignore

@@ -31,3 +31,4 @@ last_download.yaml
 roster.html
 catkin_ws/src/10-lane-control/line_detector2/include/line_detector2_tests/frame0002.jpg
 roster.html
+*.egg-info

catkin_ws/src/00-infrastructure/Dockerfile

@@ -0,0 +1,157 @@
+FROM ros:kinetic-ros-base-xenial
+MAINTAINER Andrea Censi
+
+RUN apt-get update
+
+RUN apt-get install -y \
+    software-properties-common \
+    gnupg \
+    curl \
+    git \
+    git-extras \
+    ssh \
+    pdftk \
+    bibtex2html \
+    libxml2-dev \
+    libxslt1-dev \
+    libffi6 \
+    libffi-dev \
+    libxml2-dev \
+    libxslt1-dev \
+    pdftk \
+    bibtex2html \
+    build-essential \
+    graphviz \
+    idle \
+    virtualenv \
+    python-pmw \
+    python-imaging \
+    python-yaml \
+    python-dev \
+    python-matplotlib \
+    python-numpy \
+    python-matplotlib \
+    python-setproctitle \
+    python-psutil \
+    python-lxml \
+    python-pillow \
+    python-matplotlib \
+    python-pip \
+    python-tk \
+    python-scipy \
+    python-frozendict \
+    python-tables \
+    python-sklearn \
+    python-termcolor \
+    python-setproctitle \
+    python-psutil\
+    byobu \
+    atop \
+    htop \
+    imagemagick \
+    graphviz \
+    ghostscript \
+    ntpdate  \
+    libatlas-base-dev \
+    vim\
+    apt-file\
+    iftop\
+    ros-kinetic-image-proc
+#    ros-kinetic-perception \
+#    ros-kinetic-desktop-full \
+#    ros-kinetic-tf-conversions \
+#    ros-kinetic-cv-bridge \
+#    ros-kinetic-image-transport \
+#    ros-kinetic-camera-info-manager \
+#    ros-kinetic-theora-image-transport \
+#    ros-kinetic-joy \
+#    ros-kinetic-compressed-image-transport \
+#    ros-kinetic-phidgets-drivers \
+#    ros-kinetic-imu-complementary-filter \
+#    ros-kinetic-imu-filter-madgwick \
+
+
+
+# needed for adding repository
+
+
+# Git LFS
+
+#RUN add-apt-repository -y ppa:git-core/ppa
+#RUN curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash
+#
+#RUN apt-get update
+
+#RUN apt-get install -y git-lfs \
+
+
+# Python deps
+
+RUN pip install -U \
+    empy\
+    catkin_pkg\
+    pint \
+    networkx \
+    watchdog \
+    pyramid \
+    pyramid_jinja2 \
+    pyramid_debugtoolbar \
+    bs4 \
+    nose \
+    reprep \
+    bcrypt \
+    markdown \
+    junit_xml \
+    lxml \
+    bcrypt \
+    waitress \
+    gitpython \
+    webtest \
+    chardet
+
+
+RUN add-apt-repository ppa:mc3man/xerus-media
+RUN apt-get update
+RUN apt-get install -y  mplayer mencoder ffmpeg
+# gstreamer0.10-ffmpeg
+
+
+RUN wget https://dist.ipfs.io/go-ipfs/v0.4.15/go-ipfs_v0.4.15_linux-amd64.tar.gz
+RUN tar xvzf go-ipfs_v0.4.15_linux-amd64.tar.gz
+RUN cp go-ipfs/ipfs /usr/bin
+RUN ipfs version
+
+
+RUN virtualenv --system-site-packages /project/deploy
+
+COPY requirements.txt  /project/requirements.txt
+
+RUN pip install -r /project/requirements.txt
+
+COPY . /project/10-infrastructure
+
+
+
+RUN . /project/deploy/bin/activate && cd /project/10-infrastructure && python setup.py install
+#RUN . /project/deploy/bin/activate && cd /project/10-infrastructure && python setup.py develop --no-deps
+
+RUN chmod +x /project/10-infrastructure/entrypoint.sh
+
+RUN /project/10-infrastructure/entrypoint.sh python -c "import easy_logs.cli"
+
+ENTRYPOINT ["/project/10-infrastructure/entrypoint.sh"]
+
+RUN mkdir -p \
+    /dt-data/DUCKIETOWN_ROOT \
+    /dt-data/DUCKIEFLEET_ROOT \
+    /dt-data/DUCKIETOWN_DATA \
+    /dt-data/DUCKIETOWN_TMP
+ENV DUCKIETOWN_ROOT=/dt-data/DUCKIETOWN_ROOT
+ENV DUCKIEFLEET_ROOT=/dt-data/DUCKIEFLEET_ROOT
+ENV DUCKIETOWN_DATA=/dt-data/DUCKIETOWN_DATA
+ENV DUCKIETOWN_TMP=/dt-data/DUCKIETOWN_TMP
+
+# XXX 
+RUN cp /project/10-infrastructure/easy_logs/include/easy_logs/unit_tests.urls.yaml /project/deploy/lib/python2.7/site-packages/DuckietownUtils-1.0-py2.7.egg/easy_logs
+
+RUN /project/10-infrastructure/entrypoint.sh dt-logs-summary

catkin_ws/src/00-infrastructure/Makefile

@@ -0,0 +1,29 @@
+docker_dir=.
+docker_image_name=duckietown/logs
+tag?=latest
+
+docker-build:
+	# docker build -t $(docker_image_name):$(tag) $(docker_dir)
+	docker build -t $(docker_image_name) $(docker_dir)
+
+docker-upload:
+	# docker push $(docker_image_name):$(tag)
+	docker push $(docker_image_name):latest
+
+docker-clean:
+	# Kill all running containers:
+	-docker kill $(shell sudo docker ps -q)
+
+	# Delete all stopped containers (including data-only containers):
+	-docker rm $(shell sudo docker ps -a -q)
+
+	# Delete all exited containers
+	-docker rm $(shell sudo docker ps -q -f status=exited)
+
+	# Delete all images
+	-docker rmi $(shell sudo docker images -q)
+
+
+#docker-run:
+#	docker run -it $(docker_image_name):$(tag) python -m duckietown_utils
+

catkin_ws/src/00-infrastructure/duckietown/config/baseline/calibration/kinematics/duckula.yaml

@@ -1,15 +1,6 @@
 baseline: 0.1
-<<<<<<< HEAD
 gain: 1.0
 k: 27.0
 limit: 1.0
 radius: 0.0318
 trim: 0.088
-=======
-calibration_time: 2016-04-22-20-07-15
-gain: 1.2000000476837158
-k: 27.0
-limit: 1.0
-radius: 0.0318
-trim: 0.10000000149011612
->>>>>>> dp45

catkin_ws/src/00-infrastructure/duckietown/include/duckietown_utils/bag_visualization.py

@@ -14,12 +14,13 @@
 
 @contract(returns='tuple(int, int)')
 def count_messages_in_slice(bag_filename, topic, t0, t1, stop_at=None):
-    '''
+    """
         Counts the number of messages in a slice of time.
         Stops at stop_at, if given.
 
         Returns (count, total, stopped_early), where total is the total number in the log.
-    '''
+    """
+
     bag0 = rosbag.Bag(bag_filename)
     count = bag0.get_message_count(topic_filters=[topic])
 

catkin_ws/src/00-infrastructure/duckietown/include/duckietown_utils/caching.py

@@ -13,7 +13,7 @@
 def get_cached(cache_name, f, quiet='not-given', just_delete=False):
     """
         Caches the result of f() in a file called
-            ${DUCKIETOWN_ROOT}/caches/![name].cache.pickle
+            ${DUCKIETOWN_ROOT}/caches/![cache_name].cache.pickle
     """
 
     cache_dir = get_duckietown_cache_dir()
@@ -24,6 +24,8 @@ def get_cached(cache_name, f, quiet='not-given', just_delete=False):
             logger.info('Removing %s' % cache)
             os.unlink(cache)
             return
+        else:
+            return
 
     if quiet == 'not-given':
         should_be_quiet = False

catkin_ws/src/00-infrastructure/duckietown/include/duckietown_utils/constants.py

@@ -11,8 +11,6 @@ class DuckietownConstants(object):
     DUCKIETOWN_DATA_variable = 'DUCKIETOWN_DATA'
     DUCKIETOWN_CONFIG_SEQUENCE_variable = 'DUCKIETOWN_CONFIG_SEQUENCE'
 
-#    default_download_dir = '/tmp/duckietown-download'
-
     # The name of a hypothetical robot used in the unit tests.
     ROBOT_NAME_FOR_TESTS = 'robbie'
 

catkin_ws/src/00-infrastructure/duckietown/include/duckietown_utils/disk_hierarchy.py

@@ -1,27 +1,27 @@
 import os
+from contextlib import contextmanager
 from tempfile import mkdtemp, NamedTemporaryFile
 
-from .logging_logger import logger
-
+from .constants import DuckietownConstants
 from .contracts_ import contract
 from .exception_utils import raise_desc
+from .logging_logger import logger
 from .yaml_pretty import yaml_load
-from .constants import DuckietownConstants
-
-from contextlib import contextmanager
 
 
 def mkdirs_thread_safe(dirname):
     from compmake.utils.filesystem_utils import mkdirs_thread_safe as md
     return md(dirname)
 
+
 @contract(s=str, returns=str)
 def dir_from_data(s):
     data = yaml_load(s)
     d = create_tmpdir()
     write_to_dir(data, d)
     return d
 
+
 def write_to_dir(data, d):
     if isinstance(data, dict):
         if not os.path.exists(d):
@@ -56,6 +56,7 @@ def get_dt_tmp_dir():
             pass
     return d
 
+
 def create_tmpdir(prefix='tmpdir'):
     base = get_dt_tmp_dir()
     if not os.path.exists(base):
@@ -64,6 +65,7 @@ def create_tmpdir(prefix='tmpdir'):
     d = mkdtemp(dir=base, prefix=prefix)
     return d
 
+
 @contextmanager
 def tmpfile(suffix):
     ''' Yields the name of a temporary file '''