done

.circleci/config.yml

@@ -40,6 +40,7 @@ jobs:
 
                 git clone [email protected]:duckietown/duckiefleet-fall2017.git $DUCKIEFLEET_ROOT
 
+                make test-download-logs
                 make test-circle
 
         - run:

.compmake.rc

@@ -0,0 +1 @@
+config echo 1

.gitignore

@@ -17,3 +17,6 @@ catkin_ws/src/CMakeLists.txt
 .compmake_history.txt
 t
 out-comptests/
+out-make_videos.py
+out-programmatic.py
+out-programmatic

Makefiles/Makefile.test.mk

@@ -13,6 +13,8 @@ test:
 	@echo '- `make test-comptests`:        Run the `comptests` tests.'
 	@echo '- `make test-comptests-clean`:        Run the `comptests` tests.'
 	@echo '- `make test-comptests-collect-junit`: Collects the JUnit results.'
+	@echo '- `make test-download-logs`: Downloads the logs needed for the tests.'
+
 
 test-circle: \
 	test-comptests
@@ -56,3 +58,7 @@ test-catkin_tests: check-environment
 
 test-anti_instagram: check-environment
 	bash -c "source environment.sh; rosrun anti_instagram annotation_tests.py"
+
+test-download-logs:
+	rosrun easy_logs require 20160223-amadoa-amadobot-RCDP2
+

catkin_ws/src/00-infrastructure/duckietown/include/duckietown_utils/bag_info.py

@@ -0,0 +1,31 @@
+import os
+import subprocess
+
+import yaml
+
+from duckietown_utils import logger
+
+from .caching import get_cached
+from .instantiate_utils import indent
+
+
+__all__ = ['rosbag_info', 'rosbag_info_cached']
+
+def rosbag_info_cached(filename):
+    def f():
+        return rosbag_info(filename)
+    basename = os.path.basename(filename)
+    cache_name = 'rosbag_info/' + basename
+    return get_cached(cache_name, f, quiet=True)
+
+
+
+def rosbag_info(bag):
+    stdout = subprocess.Popen(['rosbag', 'info', '--yaml', bag],
+                              stdout=subprocess.PIPE).communicate()[0]
+    try:
+        info_dict = yaml.load(stdout)
+    except:
+        logger.error('Could not parse yaml:\n%s' % indent(stdout, '| '))
+        raise
+    return info_dict

catkin_ws/src/00-infrastructure/duckietown/include/duckietown_utils/bag_reading.py

@@ -0,0 +1,35 @@
+import time
+
+from duckietown_utils import logger
+from duckietown_utils.exceptions import DTBadData
+
+
+def d8n_bag_read_with_progress(log, topic):
+    import rosbag  # @UnresolvedImport
+    bag = rosbag.Bag(log.filename)
+    length = log.length
+    n = 0
+    msg_time0 = None
+    # write a message every once in a while 
+    INTERVAL = 1
+    first = last = time.time()
+    for topic, msg, msg_time in bag.read_messages(topics=[topic]):
+        # compute progess
+        msg_time = msg_time.to_sec()
+        if msg_time0 is None:
+            msg_time0 = msg_time
+        progress = float(msg_time-msg_time0) / length
+
+        # current time
+        n += 1 
+        t = time.time()
+        if t - last > INTERVAL:
+            last =  t
+            fps =  n / (t-first) 
+            logger.debug('%6d  %4.1f%%  %5.1f fps' % (n, progress * 100, fps))
+        yield msg
+    if n == 0:
+        s = 'Could not find any message for topic %r.' % topic
+        raise DTBadData(s)
+    bag.close()
+

catkin_ws/src/00-infrastructure/duckietown/include/duckietown_utils/bag_writing.py

@@ -0,0 +1,23 @@
+from contextlib import contextmanager
+import os
+import rosbag
+
+from duckietown_utils import logger
+from duckietown_utils.mkdirs import d8n_make_sure_dir_exists
+
+
+@contextmanager
+def d8n_write_to_bag_context(out_bag_filename):
+    """
+        with d8n_write_to_bag_context(filename) as bag:
+            bag.write(topic_name, msg)
+    
+    """
+    d8n_make_sure_dir_exists(out_bag_filename)
+    out_bag = rosbag.Bag(out_bag_filename + '.tmp', 'w') 
+    yield out_bag
+    out_bag.close()
+    os.rename(out_bag_filename + '.tmp', out_bag_filename)
+    logger.info('Written bag to %s' % out_bag_filename)
+
+

catkin_ws/src/00-infrastructure/duckietown/include/duckietown_utils/constants.py

@@ -4,7 +4,6 @@
 from duckietown_utils.locate_files_impl import locate_files
 from duckietown_utils.path_utils import expand_all
 
-
 class DuckietownConstants():
     DUCKIETOWN_ROOT_variable = 'DUCKIETOWN_ROOT'
     DUCKIEFLEET_ROOT_variable = 'DUCKIEFLEET_ROOT'
@@ -14,7 +13,8 @@ class DuckietownConstants():
     # inside DUCKIEFLEET_ROOT
     scuderia_filename = 'scuderia.yaml'
     machines_path_rel_to_root = 'catkin_ws/src/00-infrastructure/duckietown/machines'
-
+    use_cache_for_algos =  False
+    use_cache_for_logs =  False
     enforce_no_tabs = True
     enforce_naming_conventions = True
 
@@ -34,6 +34,14 @@ def get_duckietown_data():
     """ Returns the path of DUCKIETOWN_DATA and checks it exists """
     return _get_dir(DuckietownConstants.DUCKIETOWN_DATA_variable)
 
+def get_duckietown_local_log_downloads():
+    """ Returns the directory to use for local downloads of logs"""
+    d = '/tmp/duckietown-download'
+    if not os.path.exists(d):
+        os.makedirs(d)
+    return d
+
+
 def get_machines_files_path():
     ''' Gets the path to the machines files. It might not exist. '''
     duckietown_root = get_duckietown_root()

catkin_ws/src/00-infrastructure/duckietown/include/duckietown_utils/exceptions.py

@@ -1,12 +1,26 @@
 
-class DTUserError(Exception):
-    """ Exceptions that will not be printed with full traceback,
+class DTException(Exception):
+    pass
+
+class DTUserError(DTException):
+    """ 
+        Exceptions that will not be printed with full traceback,
         because they contain a simple message for the user, to be printed in red.
     """
 
 class DTConfigException(DTUserError):
+    """
+        The configuration (either environment variables or YAML files)
+        is invalid.
+    """
     pass
 
+class DTBadData(DTException):
+    """
+        A log is invalid.
+    """
+
+
 import sys
 import traceback
 

catkin_ws/src/00-infrastructure/duckietown/include/duckietown_utils/friendly_path_imp.py

@@ -40,12 +40,13 @@ def friendly_path(path, use_environment=True):
             if 'PWD' in e:
                 del envs[e]
 
-        for k, v in envs.items():
-            if v:
-                if v and v[-1] == '/':
-                    v = v[:-1]
-                if v[0] == '/':
-                    rules.append(('${%s}' % k, v))
+        for k, v0 in envs.items():
+            if v0:
+                for v in [v0, os.path.realpath(v0)]:
+                    if v and v[-1] == '/':
+                        v = v[:-1]
+                    if v[0] == '/':
+                        rules.append(('${%s}' % k, v))
 
     # apply longest first
     rules.sort(key=lambda x: (-len(x[1])))

catkin_ws/src/00-infrastructure/duckietown/include/duckietown_utils/fuzzy.py

@@ -188,7 +188,7 @@ def parse_match_spec(s):
 def fuzzy_match(spec, stuff):
     """
         spec: a string
-        logs: a dict of logs
+        logs: an OrderedDict str -> object
     """
     if not isinstance(stuff, dict):
         msg = 'Expectd an OrderedDict, got %s.' % describe_type(stuff)

catkin_ws/src/00-infrastructure/duckietown/include/duckietown_utils/image_conversions.py

@@ -0,0 +1,24 @@
+from cv_bridge import CvBridge  # @UnresolvedImport
+
+__all__ = ['d8n_image_msg_from_cv_image']
+
+class ImageConversions():
+
+    bridge = None
+
+def d8n_image_msg_from_cv_image(cv_image, image_format, same_timestamp_as = None):
+    """ 
+        Makes an Image message from a CV image. 
+    
+        if same_timestamp_as is not None, we copy the timestamp
+        from that image.
+        
+        image_format: 'bgr8' or 'mono' or similar
+    """
+    if ImageConversions.bridge is None:
+        ImageConversions.bridge = CvBridge()
+    image_msg_out = ImageConversions.bridge.cv2_to_imgmsg(cv_image, image_format)
+    if same_timestamp_as is not None:
+        image_msg_out.header.stamp = same_timestamp_as.header.stamp
+    return image_msg_out
+

catkin_ws/src/00-infrastructure/duckietown/include/duckietown_utils/mkdirs.py

@@ -0,0 +1,36 @@
+# copied from compmake
+
+import os
+
+__all__ = [
+    'make_sure_dir_exists',
+    'mkdirs_thread_safe',
+]
+
+
+def d8n_make_sure_dir_exists(filename):
+    """ 
+        Makes sure that the path to file exists, by creating directories. 
+        
+    """
+    dirname = os.path.dirname(filename)
+    # dir == '' for current dir
+    if dirname != '' and not os.path.exists(dirname):
+        mkdirs_thread_safe(dirname)
+
+
+def mkdirs_thread_safe(dst):
+    """ Make directories leading to 'dst' if they don't exist yet"""
+    if dst == '' or os.path.exists(dst):
+        return
+    head, _ = os.path.split(dst)
+    if os.sep == ':' and not ':' in head:
+        head += ':'
+    mkdirs_thread_safe(head)
+    try:
+        mode = 511  # 0777 in octal
+        os.mkdir(dst, mode)
+    except OSError as err:
+        if err.errno != 17:  # file exists
+            raise
+

catkin_ws/src/00-infrastructure/duckietown/include/duckietown_utils/yaml_wrap.py

@@ -5,7 +5,7 @@
 
 from duckietown_utils import logger
 from duckietown_utils.constants import get_catkin_ws_src, get_duckiefleet_root,\
-    get_duckietown_data
+    get_duckietown_data, get_duckietown_local_log_downloads
 from duckietown_utils.exception_utils import raise_wrapped
 from duckietown_utils.exceptions import DTConfigException
 from duckietown_utils.instantiate_utils import indent
@@ -80,6 +80,10 @@ def look_everywhere_for_bag_files(pattern='*.bag'):
     # then we look in $DUCKIETOWN_FLEET
     sources.append(get_duckiefleet_root())
     sources.append(get_duckietown_data())
+    # downloads 
+    p = get_duckietown_local_log_downloads()
+    if os.path.exists(p):
+        sources.append(p)
 
     logger.info('Looking for files with pattern %s\n' % pattern)
 

catkin_ws/src/00-infrastructure/easy_algo/include/easy_algo/algo_db.py

@@ -13,14 +13,16 @@
     get_config_sources
 
 from .algo_structures import EasyAlgoTest, EasyAlgoInstance, EasyAlgoFamily
+from duckietown_utils.fuzzy import fuzzy_match
+from duckietown_utils.constants import DuckietownConstants
 
 
 __all__ = ['get_easy_algo_db', 'EasyAlgoDB']
-cache_algos = False
+
 
 def get_easy_algo_db():
     if EasyAlgoDB._singleton is None:
-
+        cache_algos = DuckietownConstants.use_cache_for_algos
         EasyAlgoDB._singleton = get_cached('EasyAlgoDB', EasyAlgoDB) if cache_algos else EasyAlgoDB()
     return EasyAlgoDB._singleton
 
@@ -35,6 +37,11 @@ def __init__(self, sources=None):
             sources = get_config_sources()
         self.family_name2config = load_family_config(sources)
 
+    def query(self, family_name, query):
+        family = self.get_family(family_name)
+        instances = family.instances
+        return fuzzy_match(query, instances)
+
     def get_family(self, x):
         check_is_in('family', x, self.family_name2config)
         return self.family_name2config[x]
@@ -234,8 +241,6 @@ def interpret_easy_algo_config(filename, data):
                             valid=True,
                             error_if_invalid=False)
 
-
-
-
+