From 5de03f490d3c8e5ab26de203235b1726283db5a2 Mon Sep 17 00:00:00 2001 From: Stephen Shirley Date: Tue, 21 Nov 2017 12:43:37 +0100 Subject: [PATCH] Don't mix generated and non-generated files in bin/ (#1332) Python executable wrappers are moved into python/bin/. Also: - Remove topo generator support for the now-removed python router, including the support for `topology.yaml`. - Fix the shebang in scion.sh to be consistent. Fixes #1259 --- .gitignore | 14 ++-- Makefile | 1 + bin/.keepme | 0 {bin => python/bin}/beacon_server | 0 {bin => python/bin}/cert_server | 0 {bin => python/bin}/path_server | 0 {bin => python/bin}/sciond | 0 {bin => python/bin}/sibra_server | 0 python/topology/generator.py | 131 +++--------------------------- scion.sh | 4 +- 10 files changed, 19 insertions(+), 131 deletions(-) create mode 100644 bin/.keepme rename {bin => python/bin}/beacon_server (100%) rename {bin => python/bin}/cert_server (100%) rename {bin => python/bin}/path_server (100%) rename {bin => python/bin}/sciond (100%) rename {bin => python/bin}/sibra_server (100%) diff --git a/.gitignore b/.gitignore index 29d47ad858..3fca9e7612 100644 --- a/.gitignore +++ b/.gitignore @@ -59,16 +59,12 @@ topology/mininet/*.pyc *.o *.so -# Executable binaries +# Generated executables ########################## -bin/border -bin/dispatcher -bin/logdog -bin/sig -bin/topopruner -bin/echoscion -c/dispatcher/dispatcher -c/lib/scion/checksum_bench +/bin/* +!/bin/.keepme +/c/dispatcher/dispatcher +/c/lib/scion/checksum_bench # CTags ########################## diff --git a/Makefile b/Makefile index e75b1e1b5e..591f734567 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,7 @@ all: clibs dispatcher go clean: $(foreach var,$(SRC_DIRS),$(MAKE) -C $(var) clean || exit 1;) cd go && $(MAKE) clean + rm -f bin/* rm -f tags go: libscion diff --git a/bin/.keepme b/bin/.keepme new file mode 100644 index 0000000000..e69de29bb2 diff --git a/bin/beacon_server b/python/bin/beacon_server similarity index 100% rename from bin/beacon_server rename to python/bin/beacon_server diff --git a/bin/cert_server b/python/bin/cert_server similarity index 100% rename from bin/cert_server rename to python/bin/cert_server diff --git a/bin/path_server b/python/bin/path_server similarity index 100% rename from bin/path_server rename to python/bin/path_server diff --git a/bin/sciond b/python/bin/sciond similarity index 100% rename from bin/sciond rename to python/bin/sciond diff --git a/bin/sibra_server b/python/bin/sibra_server similarity index 100% rename from bin/sibra_server rename to python/bin/sibra_server diff --git a/python/topology/generator.py b/python/topology/generator.py index b75ee00597..61ec40ffa3 100755 --- a/python/topology/generator.py +++ b/python/topology/generator.py @@ -115,7 +115,6 @@ DEFAULT_NETWORK = "127.0.0.0/8" DEFAULT_PRIV_NETWORK = "192.168.0.0/16" DEFAULT_MININET_NETWORK = "100.64.0.0/10" -DEFAULT_ROUTER = "go" SCION_SERVICE_NAMES = ( "BeaconService", @@ -127,7 +126,6 @@ DEFAULT_KEYGEN_ALG = 'ed25519' -GENERATE_BOTH_TOPOLOGY = True GENERATE_BIND_ADDRESS = False @@ -138,7 +136,7 @@ class ConfigGenerator(object): def __init__(self, out_dir=GEN_PATH, topo_file=DEFAULT_TOPOLOGY_FILE, path_policy_file=DEFAULT_PATH_POLICY_FILE, zk_config_file=DEFAULT_ZK_CONFIG, network=None, - use_mininet=False, router="py", bind_addr=GENERATE_BIND_ADDRESS, + use_mininet=False, bind_addr=GENERATE_BIND_ADDRESS, pseg_ttl=DEFAULT_SEGMENT_TTL): """ Initialize an instance of the class ConfigGenerator. @@ -159,7 +157,6 @@ def __init__(self, out_dir=GEN_PATH, topo_file=DEFAULT_TOPOLOGY_FILE, self.mininet = use_mininet self.default_zookeepers = {} self.default_mtu = None - self.router = router self.gen_bind_addr = bind_addr self.pseg_ttl = pseg_ttl self._read_defaults(network) @@ -221,8 +218,7 @@ def _generate_topology(self): def _generate_supervisor(self, topo_dicts, zookeepers): super_gen = SupervisorGenerator( - self.out_dir, topo_dicts, zookeepers, self.zk_config, self.mininet, - self.router) + self.out_dir, topo_dicts, zookeepers, self.zk_config, self.mininet) super_gen.generate() def _generate_zk_conf(self, zookeepers): @@ -694,12 +690,6 @@ def _write_as_topos(self): contents_json = json.dumps(self.topo_dicts[topo_id], default=_json_default, indent=2) write_file(path, contents_json) - if GENERATE_BOTH_TOPOLOGY: - path_yaml = os.path.join(base, "topology.yml") - topo_dicts_old = _topo_json_to_yaml(self.topo_dicts[topo_id]) - contents_yaml = yaml.dump(topo_dicts_old, - default_flow_style=False) - write_file(path_yaml, contents_yaml) # Test if topo file parses cleanly Topology.from_file(path) @@ -783,14 +773,12 @@ def _write_target_file(self, base_path, target_addrs, ele_type): class SupervisorGenerator(object): - def __init__(self, out_dir, topo_dicts, zookeepers, zk_config, mininet, - router): + def __init__(self, out_dir, topo_dicts, zookeepers, zk_config, mininet): self.out_dir = out_dir self.topo_dicts = topo_dicts self.zookeepers = zookeepers self.zk_config = zk_config self.mininet = mininet - self.router = router def generate(self): self._write_dispatcher_conf() @@ -801,17 +789,13 @@ def _as_conf(self, topo_id, topo): entries = [] base = self._get_base_path(topo_id) for key, cmd in ( - ("BeaconService", "bin/beacon_server"), - ("CertificateService", "bin/cert_server"), - ("PathService", "bin/path_server"), - ("SibraService", "bin/sibra_server"), + ("BeaconService", "python/bin/beacon_server"), + ("CertificateService", "python/bin/cert_server"), + ("PathService", "python/bin/path_server"), + ("SibraService", "python/bin/sibra_server"), ): entries.extend(self._std_entries(topo, key, cmd, base)) - if self.router == "go": - entries.extend(self._br_entries(topo, "bin/border", base)) - else: - entries.extend(self._std_entries(topo, "BorderRouters", - "bin/router", base)) + entries.extend(self._br_entries(topo, "bin/border", base)) entries.extend(self._zk_entries(topo_id)) self._write_as_conf(topo_id, entries) @@ -833,7 +817,7 @@ def _br_entries(self, topo, cmd, base): def _sciond_entry(self, name, conf_dir): path = self._sciond_path(name) return self._common_entry( - name, ["bin/sciond", "--api-addr", path, name, conf_dir]) + name, ["python/bin/sciond", "--api-addr", path, name, conf_dir]) def _sciond_path(self, name): return os.path.join(SCIOND_API_SOCKDIR, "%s.sock" % name) @@ -894,7 +878,7 @@ def _write_elem_conf(self, elem, entry, elem_dir, topo_id=None): # Else set the SCIOND_PATH env to point to the per-AS sciond. path = self._sciond_path("sd%s" % topo_id) prog['environment'] += ',SCIOND_PATH="%s"' % path - if elem.startswith("br") and self.router == "go": + if elem.startswith("br"): prog['environment'] += ',GODEBUG="cgocheck=0"' if elem.startswith("zk") and self.zk_config["Environment"]: for k, v in self.zk_config["Environment"].items(): @@ -1190,97 +1174,6 @@ def _json_default(o): raise TypeError -def _topo_json_to_yaml(topo_dicts): - """ - Convert the new topology format (which uses json) to the old format (which uses yaml). - XXX(kormat): This is only needed until the BR is switched over to the new topo format. - - :pram dict topo_dicts: new topology dict - :return dict topo_old: old topology dict - """ - topo_old = {} - for service, attributes in topo_dicts.items(): - if service == "Overlay": - continue - elif service == "BeaconService": - topo_old["BeaconServers"] = {} - for bs_id, entity in attributes.items(): - bs_addr = topo_dicts[service][bs_id]["Public"][0]["Addr"] - bs_port = topo_dicts[service][bs_id]["Public"][0]["L4Port"] - topo_old["BeaconServers"][bs_id] = { - 'Addr': bs_addr, - 'Port': bs_port - } - elif service == "BorderRouters": - topo_old["BorderRouters"] = {} - for br_id, entity in attributes.items(): - br_addr = topo_dicts[service][br_id]["InternalAddrs"][0]["Public"][0]["Addr"] - br_port = topo_dicts[service][br_id]["InternalAddrs"][0]["Public"][0]["L4Port"] - for ifid, temp in topo_dicts[service][br_id]["Interfaces"].items(): - if_addr = topo_dicts[service][br_id]["Interfaces"][ifid]["Public"]["Addr"] - if_port = topo_dicts[service][br_id]["Interfaces"][ifid]["Public"]["L4Port"] - bandwidth = topo_dicts[service][br_id]["Interfaces"][ifid]["Bandwidth"] - isdas = topo_dicts[service][br_id]["Interfaces"][ifid]["ISD_AS"] - linktype = topo_dicts[service][br_id]["Interfaces"][ifid]["LinkType"] - mtu = topo_dicts[service][br_id]["Interfaces"][ifid]["MTU"] - toaddr = topo_dicts[service][br_id]["Interfaces"][ifid]["Remote"]["Addr"] - toport = topo_dicts[service][br_id]["Interfaces"][ifid]["Remote"]["L4Port"] - topo_old[service][br_id] = { - 'Addr': br_addr, - 'Port': br_port, - 'Interface': { - 'Addr': if_addr, - 'UdpPort': if_port, - 'IFID': ifid, - 'Bandwidth': bandwidth, - 'ISD_AS': isdas, - 'LinkType': linktype, - 'MTU': mtu, - 'ToAddr': toaddr, - 'ToUdpPort': toport, - } - } - elif service == "CertificateService": - topo_old["CertificateServers"] = {} - for cs_id, entity in attributes.items(): - cs_addr = topo_dicts[service][cs_id]["Public"][0]["Addr"] - cs_port = topo_dicts[service][cs_id]["Public"][0]["L4Port"] - topo_old["CertificateServers"][cs_id] = { - 'Addr': cs_addr, - 'Port': cs_port - } - elif service == "PathService": - topo_old["PathServers"] = {} - for ps_id, entity in attributes.items(): - ps_addr = topo_dicts[service][ps_id]["Public"][0]["Addr"] - ps_port = topo_dicts[service][ps_id]["Public"][0]["L4Port"] - topo_old["PathServers"][ps_id] = { - 'Addr': ps_addr, - 'Port': ps_port - } - elif service == "SibraService": - topo_old["SibraServers"] = {} - for sb_id, entity in attributes.items(): - sb_addr = topo_dicts[service][sb_id]["Public"][0]["Addr"] - sb_port = topo_dicts[service][sb_id]["Public"][0]["L4Port"] - topo_old["SibraServers"][sb_id] = { - 'Addr': sb_addr, - 'Port': sb_port - } - elif service == "ZookeeperService": - topo_old["Zookeepers"] = {} - for zk_id, entity in attributes.items(): - zk_addr = topo_dicts[service][zk_id]["Addr"] - zk_port = topo_dicts[service][zk_id]["L4Port"] - topo_old["Zookeepers"][zk_id] = { - 'Addr': zk_addr, - 'Port': zk_port - } - else: - topo_old[service] = attributes - return topo_old - - def _prom_addr_br(br_ele): """Get the prometheus address for a border router""" int_addr = br_ele['InternalAddrs'][0]['Public'][0] @@ -1310,8 +1203,6 @@ def main(): help='Output directory') parser.add_argument('-z', '--zk-config', default=DEFAULT_ZK_CONFIG, help='Zookeeper configuration file') - parser.add_argument('-r', '--router', default=DEFAULT_ROUTER, - help='Router implementation to use ("go" or "py")') parser.add_argument('-b', '--bind-addr', default=GENERATE_BIND_ADDRESS, help='Generate bind addresses (E.g. "192.168.0.0/16"') parser.add_argument('--pseg-ttl', type=int, default=DEFAULT_SEGMENT_TTL, @@ -1319,7 +1210,7 @@ def main(): args = parser.parse_args() confgen = ConfigGenerator( args.output_dir, args.topo_config, args.path_policy, args.zk_config, - args.network, args.mininet, args.router, args.bind_addr, args.pseg_ttl) + args.network, args.mininet, args.bind_addr, args.pseg_ttl) confgen.generate_all() diff --git a/scion.sh b/scion.sh index 94417c44db..8e6b66f246 100755 --- a/scion.sh +++ b/scion.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/bin/bash export PYTHONPATH=python/:. @@ -149,7 +149,7 @@ cmd_sciond() { GENDIR=gen/ISD${ISD}/AS${AS}/endhost [ -d "$GENDIR" ] || { echo "Topology directory for $ISD-$AS doesn't exist: $GENDIR"; exit 1; } APIADDR="/run/shm/sciond/${ISD}-${AS}.sock" - PYTHONPATH=python/:. bin/sciond --addr $ADDR --api-addr $APIADDR sd${ISD}-${AS} $GENDIR & + PYTHONPATH=python/:. python/bin/sciond --addr $ADDR --api-addr $APIADDR sd${ISD}-${AS} $GENDIR & echo "Sciond running for $ISD-$AS (pid $!)" wait exit $?