Don't mix generated and non-generated files in bin/ (#1332)

Python executable wrappers are moved into python/bin/.

Also:
- Remove topo generator support for the now-removed python router,
  including the support for `topology.yaml`.
- Fix the shebang in scion.sh to be consistent.

Fixes #1259

.gitignore

@@ -59,16 +59,12 @@ topology/mininet/*.pyc
 *.o
 *.so
 
-# Executable binaries
+# Generated executables
 ##########################
-bin/border
-bin/dispatcher
-bin/logdog
-bin/sig
-bin/topopruner
-bin/echoscion
-c/dispatcher/dispatcher
-c/lib/scion/checksum_bench
+/bin/*
+!/bin/.keepme
+/c/dispatcher/dispatcher
+/c/lib/scion/checksum_bench
 
 # CTags
 ##########################

Makefile

@@ -7,6 +7,7 @@ all: clibs dispatcher go
 clean:
 	$(foreach var,$(SRC_DIRS),$(MAKE) -C $(var) clean || exit 1;)
 	cd go && $(MAKE) clean
+	rm -f bin/*
 	rm -f tags
 
 go: libscion

python/topology/generator.py

@@ -115,7 +115,6 @@
 DEFAULT_NETWORK = "127.0.0.0/8"
 DEFAULT_PRIV_NETWORK = "192.168.0.0/16"
 DEFAULT_MININET_NETWORK = "100.64.0.0/10"
-DEFAULT_ROUTER = "go"
 
 SCION_SERVICE_NAMES = (
     "BeaconService",
@@ -127,7 +126,6 @@
 
 DEFAULT_KEYGEN_ALG = 'ed25519'
 
-GENERATE_BOTH_TOPOLOGY = True
 GENERATE_BIND_ADDRESS = False
 
 
@@ -138,7 +136,7 @@ class ConfigGenerator(object):
     def __init__(self, out_dir=GEN_PATH, topo_file=DEFAULT_TOPOLOGY_FILE,
                  path_policy_file=DEFAULT_PATH_POLICY_FILE,
                  zk_config_file=DEFAULT_ZK_CONFIG, network=None,
-                 use_mininet=False, router="py", bind_addr=GENERATE_BIND_ADDRESS,
+                 use_mininet=False, bind_addr=GENERATE_BIND_ADDRESS,
                  pseg_ttl=DEFAULT_SEGMENT_TTL):
         """
         Initialize an instance of the class ConfigGenerator.
@@ -159,7 +157,6 @@ def __init__(self, out_dir=GEN_PATH, topo_file=DEFAULT_TOPOLOGY_FILE,
         self.mininet = use_mininet
         self.default_zookeepers = {}
         self.default_mtu = None
-        self.router = router
         self.gen_bind_addr = bind_addr
         self.pseg_ttl = pseg_ttl
         self._read_defaults(network)
@@ -221,8 +218,7 @@ def _generate_topology(self):
 
     def _generate_supervisor(self, topo_dicts, zookeepers):
         super_gen = SupervisorGenerator(
-            self.out_dir, topo_dicts, zookeepers, self.zk_config, self.mininet,
-            self.router)
+            self.out_dir, topo_dicts, zookeepers, self.zk_config, self.mininet)
         super_gen.generate()
 
     def _generate_zk_conf(self, zookeepers):
@@ -694,12 +690,6 @@ def _write_as_topos(self):
             contents_json = json.dumps(self.topo_dicts[topo_id],
                                        default=_json_default, indent=2)
             write_file(path, contents_json)
-            if GENERATE_BOTH_TOPOLOGY:
-                path_yaml = os.path.join(base, "topology.yml")
-                topo_dicts_old = _topo_json_to_yaml(self.topo_dicts[topo_id])
-                contents_yaml = yaml.dump(topo_dicts_old,
-                                          default_flow_style=False)
-                write_file(path_yaml, contents_yaml)
             # Test if topo file parses cleanly
             Topology.from_file(path)
 
@@ -783,14 +773,12 @@ def _write_target_file(self, base_path, target_addrs, ele_type):
 
 
 class SupervisorGenerator(object):
-    def __init__(self, out_dir, topo_dicts, zookeepers, zk_config, mininet,
-                 router):
+    def __init__(self, out_dir, topo_dicts, zookeepers, zk_config, mininet):
         self.out_dir = out_dir
         self.topo_dicts = topo_dicts
         self.zookeepers = zookeepers
         self.zk_config = zk_config
         self.mininet = mininet
-        self.router = router
 
     def generate(self):
         self._write_dispatcher_conf()
@@ -801,17 +789,13 @@ def _as_conf(self, topo_id, topo):
         entries = []
         base = self._get_base_path(topo_id)
         for key, cmd in (
-            ("BeaconService", "bin/beacon_server"),
-            ("CertificateService", "bin/cert_server"),
-            ("PathService", "bin/path_server"),
-            ("SibraService", "bin/sibra_server"),
+            ("BeaconService", "python/bin/beacon_server"),
+            ("CertificateService", "python/bin/cert_server"),
+            ("PathService", "python/bin/path_server"),
+            ("SibraService", "python/bin/sibra_server"),
         ):
             entries.extend(self._std_entries(topo, key, cmd, base))
-        if self.router == "go":
-            entries.extend(self._br_entries(topo, "bin/border", base))
-        else:
-            entries.extend(self._std_entries(topo, "BorderRouters",
-                                             "bin/router", base))
+        entries.extend(self._br_entries(topo, "bin/border", base))
         entries.extend(self._zk_entries(topo_id))
         self._write_as_conf(topo_id, entries)
 
@@ -833,7 +817,7 @@ def _br_entries(self, topo, cmd, base):
     def _sciond_entry(self, name, conf_dir):
         path = self._sciond_path(name)
         return self._common_entry(
-            name, ["bin/sciond", "--api-addr", path, name, conf_dir])
+            name, ["python/bin/sciond", "--api-addr", path, name, conf_dir])
 
     def _sciond_path(self, name):
         return os.path.join(SCIOND_API_SOCKDIR, "%s.sock" % name)
@@ -894,7 +878,7 @@ def _write_elem_conf(self, elem, entry, elem_dir, topo_id=None):
                 # Else set the SCIOND_PATH env to point to the per-AS sciond.
                 path = self._sciond_path("sd%s" % topo_id)
                 prog['environment'] += ',SCIOND_PATH="%s"' % path
-        if elem.startswith("br") and self.router == "go":
+        if elem.startswith("br"):
             prog['environment'] += ',GODEBUG="cgocheck=0"'
         if elem.startswith("zk") and self.zk_config["Environment"]:
             for k, v in self.zk_config["Environment"].items():
@@ -1190,97 +1174,6 @@ def _json_default(o):
     raise TypeError
 
 
-def _topo_json_to_yaml(topo_dicts):
-    """
-    Convert the new topology format (which uses json) to the old format (which uses yaml).
-    XXX(kormat): This is only needed until the BR is switched over to the new topo format.
-
-    :pram dict topo_dicts: new topology dict
-    :return dict topo_old: old topology dict
-    """
-    topo_old = {}
-    for service, attributes in topo_dicts.items():
-        if service == "Overlay":
-            continue
-        elif service == "BeaconService":
-            topo_old["BeaconServers"] = {}
-            for bs_id, entity in attributes.items():
-                bs_addr = topo_dicts[service][bs_id]["Public"][0]["Addr"]
-                bs_port = topo_dicts[service][bs_id]["Public"][0]["L4Port"]
-                topo_old["BeaconServers"][bs_id] = {
-                    'Addr': bs_addr,
-                    'Port': bs_port
-                }
-        elif service == "BorderRouters":
-            topo_old["BorderRouters"] = {}
-            for br_id, entity in attributes.items():
-                br_addr = topo_dicts[service][br_id]["InternalAddrs"][0]["Public"][0]["Addr"]
-                br_port = topo_dicts[service][br_id]["InternalAddrs"][0]["Public"][0]["L4Port"]
-                for ifid, temp in topo_dicts[service][br_id]["Interfaces"].items():
-                    if_addr = topo_dicts[service][br_id]["Interfaces"][ifid]["Public"]["Addr"]
-                    if_port = topo_dicts[service][br_id]["Interfaces"][ifid]["Public"]["L4Port"]
-                    bandwidth = topo_dicts[service][br_id]["Interfaces"][ifid]["Bandwidth"]
-                    isdas = topo_dicts[service][br_id]["Interfaces"][ifid]["ISD_AS"]
-                    linktype = topo_dicts[service][br_id]["Interfaces"][ifid]["LinkType"]
-                    mtu = topo_dicts[service][br_id]["Interfaces"][ifid]["MTU"]
-                    toaddr = topo_dicts[service][br_id]["Interfaces"][ifid]["Remote"]["Addr"]
-                    toport = topo_dicts[service][br_id]["Interfaces"][ifid]["Remote"]["L4Port"]
-                    topo_old[service][br_id] = {
-                        'Addr': br_addr,
-                        'Port': br_port,
-                        'Interface': {
-                            'Addr': if_addr,
-                            'UdpPort': if_port,
-                            'IFID': ifid,
-                            'Bandwidth': bandwidth,
-                            'ISD_AS': isdas,
-                            'LinkType': linktype,
-                            'MTU': mtu,
-                            'ToAddr': toaddr,
-                            'ToUdpPort': toport,
-                        }
-                    }
-        elif service == "CertificateService":
-            topo_old["CertificateServers"] = {}
-            for cs_id, entity in attributes.items():
-                cs_addr = topo_dicts[service][cs_id]["Public"][0]["Addr"]
-                cs_port = topo_dicts[service][cs_id]["Public"][0]["L4Port"]
-                topo_old["CertificateServers"][cs_id] = {
-                    'Addr': cs_addr,
-                    'Port': cs_port
-                }
-        elif service == "PathService":
-            topo_old["PathServers"] = {}
-            for ps_id, entity in attributes.items():
-                ps_addr = topo_dicts[service][ps_id]["Public"][0]["Addr"]
-                ps_port = topo_dicts[service][ps_id]["Public"][0]["L4Port"]
-                topo_old["PathServers"][ps_id] = {
-                    'Addr': ps_addr,
-                    'Port': ps_port
-                }
-        elif service == "SibraService":
-            topo_old["SibraServers"] = {}
-            for sb_id, entity in attributes.items():
-                sb_addr = topo_dicts[service][sb_id]["Public"][0]["Addr"]
-                sb_port = topo_dicts[service][sb_id]["Public"][0]["L4Port"]
-                topo_old["SibraServers"][sb_id] = {
-                    'Addr': sb_addr,
-                    'Port': sb_port
-                }
-        elif service == "ZookeeperService":
-            topo_old["Zookeepers"] = {}
-            for zk_id, entity in attributes.items():
-                zk_addr = topo_dicts[service][zk_id]["Addr"]
-                zk_port = topo_dicts[service][zk_id]["L4Port"]
-                topo_old["Zookeepers"][zk_id] = {
-                    'Addr': zk_addr,
-                    'Port': zk_port
-                }
-        else:
-            topo_old[service] = attributes
-    return topo_old
-
-
 def _prom_addr_br(br_ele):
     """Get the prometheus address for a border router"""
     int_addr = br_ele['InternalAddrs'][0]['Public'][0]
@@ -1310,16 +1203,14 @@ def main():
                         help='Output directory')
     parser.add_argument('-z', '--zk-config', default=DEFAULT_ZK_CONFIG,
                         help='Zookeeper configuration file')
-    parser.add_argument('-r', '--router', default=DEFAULT_ROUTER,
-                        help='Router implementation to use ("go" or "py")')
     parser.add_argument('-b', '--bind-addr', default=GENERATE_BIND_ADDRESS,
                         help='Generate bind addresses (E.g. "192.168.0.0/16"')
     parser.add_argument('--pseg-ttl', type=int, default=DEFAULT_SEGMENT_TTL,
                         help='Path segment TTL (in seconds)')
     args = parser.parse_args()
     confgen = ConfigGenerator(
         args.output_dir, args.topo_config, args.path_policy, args.zk_config,
-        args.network, args.mininet, args.router, args.bind_addr, args.pseg_ttl)
+        args.network, args.mininet, args.bind_addr, args.pseg_ttl)
     confgen.generate_all()
 
 

scion.sh

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
 
 export PYTHONPATH=python/:.
 
@@ -149,7 +149,7 @@ cmd_sciond() {
     GENDIR=gen/ISD${ISD}/AS${AS}/endhost
     [ -d "$GENDIR" ] || { echo "Topology directory for $ISD-$AS doesn't exist: $GENDIR"; exit 1; }
     APIADDR="/run/shm/sciond/${ISD}-${AS}.sock"
-    PYTHONPATH=python/:. bin/sciond --addr $ADDR --api-addr $APIADDR sd${ISD}-${AS} $GENDIR &
+    PYTHONPATH=python/:. python/bin/sciond --addr $ADDR --api-addr $APIADDR sd${ISD}-${AS} $GENDIR &
     echo "Sciond running for $ISD-$AS (pid $!)"
     wait
     exit $?