diff --git a/.gitignore b/.gitignore index c8de2e7..b392570 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,11 @@ +.vscode/ +build/ +**egg-info** +nano_nodes/ **/*pycache* **/*_cache* .old* config.json venv_py/ **/.DS_Store -**/._.DS_Store \ No newline at end of file +**/._.DS_Store diff --git a/.vscode/launch.json b/.vscode/launch.json deleted file mode 100644 index f5a796c..0000000 --- a/.vscode/launch.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "version": "0.2.0", - "configurations": [ - { - "name": "Python: Run run.py", - "type": "python", - "request": "launch", - "program": "${workspaceFolder}/run.py", - "console": "integratedTerminal", - "cwd": "${workspaceFolder}", - "env": { - "SNIPPET_FILE": "app/snippets.json", - "CONFIG_FILE" : "app/configs/3node_network/easy_setup.json" - } - } - ] -} diff --git a/app/pycmd.py b/app/pycmd.py deleted file mode 100644 index 436b6b5..0000000 --- a/app/pycmd.py +++ /dev/null @@ -1,34 +0,0 @@ -import time -import app.node_interaction as nni -from app.node_tools import StatsLogger -import asyncio - -statsLogger = None - - -class NodeCommands: - pass - - -class TestClass: - - def long_running_method(self, duration=1): - time.sleep(duration) - - -class NodeInteraction: - - def __init__(self): - # Your existing code... - self.logger = None - - def publish_blocks_test(self, - params, - logger_type="rpc", - logger_timeout=600, - logger_include_peers=None, - logger_exclude_peers=None): - - asyncio.run( - nni.xnolib_publish(params, logger_type, logger_timeout, - logger_include_peers, logger_exclude_peers)) diff --git a/submodules/xnomin/__init__.py b/nanolab/__init__.py similarity index 100% rename from submodules/xnomin/__init__.py rename to nanolab/__init__.py diff --git a/nanolab/configs/10node_network/change_blocks.json b/nanolab/configs/10node_network/change_blocks.json new file mode 100644 index 0000000..ed418d6 --- /dev/null +++ b/nanolab/configs/10node_network/change_blocks.json @@ -0,0 +1,115 @@ +{ + "_docker_tags": [ "db35_rev_a302_fde8", "nanocurrency/nano-beta:V25.0DB35", "nanocurrency/nano:V24.0" ], + "docker_tags": [ + "nanocurrency/nano-beta:V25.0DB35" + ], + "commands": [ + { + "skip": false, + "type": "snippet", + "key": "setup_ledger_and_config", + "variables": { + "NL_CONFIG": "nanolab/configs/10node_network/nl_config_default.toml", + "REMOTE_IP": "127.0.0.1", + "SETUP_NODES": "nl_genesis nl_pr1 nl_pr2 nl_pr3 nl_pr4 nl_pr5 nl_pr6 nl_pr7 nl_pr8 nl_pr9 nl_pr10", + "LEDGER": "nanolab/data/ledgers/10pr_bucket0-1-88-90-100_10kaccs.ldb" + } + }, + { + "skip" : false, + "type" : "threaded", + "commands" : [ + { + "skip": false, + "type": "python", + "method": "publish_blocks", + "variables": { + "publish_params": { + "blocks_path": "nanolab/data/blocks/10node_bucket_rounds.json", + "bps": 1000, + "start_round" : 0, + "end_round" : 1, + "subset" :{ + "start_index" : 0, + "end_index" : 12500 + } + }, + "logger_type": "rpc", + "logger_expected_count" : 50000, + "logger_timeout": 180, + "_logger_include_peers": ["nl_pr1"] + } + },{ + "skip": false, + "type": "python", + "method": "publish_blocks", + "variables": { + "publish_params": { + "blocks_path": "nanolab/data/blocks/10node_bucket_rounds.json", + "bps": 1000, + "start_round" : 0, + "end_round" : 1, + "subset" :{ + "start_index" : 12500, + "end_index" : 25000 + } + } + } + },{ + "skip": false, + "type": "python", + "method": "publish_blocks", + "variables": { + "publish_params": { + "blocks_path": "nanolab/data/blocks/10node_bucket_rounds.json", + "bps": 1000, + "start_round" : 0, + "end_round" : 1, + "subset" :{ + "start_index" : 25000, + "end_index" : 37500 + } + } + } + },{ + "skip": false, + "type": "python", + "method": "publish_blocks", + "variables": { + "publish_params": { + "blocks_path": "nanolab/data/blocks/10node_bucket_rounds.json", + "bps": 1000, + "start_round" : 0, + "end_round" : 1, + "subset" :{ + "start_index" : 37500, + "end_index" : 50000 + } + } + } + } + ] + }, + { + "skip": true, + "type": "python", + "method": "publish_blocks", + "variables": { + "publish_params": { + "blocks_path": "nanolab/data/blocks/10node_bucket_rounds.json", + "bps": 1000, + "start_round" : 0, + "end_round" : 1, + "subset" :{ + "start_index" : 0, + "end_index" : 1000 + } + + }, + "logger_type": "rpc", + "logger_timeout": 30, + "logger_include_peers": ["nl_pr1"] + } + } + ] +} \ No newline at end of file diff --git a/nanolab/configs/10node_network/easy_setup.json b/nanolab/configs/10node_network/easy_setup.json new file mode 100644 index 0000000..f5ecf38 --- /dev/null +++ b/nanolab/configs/10node_network/easy_setup.json @@ -0,0 +1,36 @@ +{ + "docker_tags": [ + "nanocurrency/nano-beta:V25.0DB35", "nanocurrency/nano:V24.0", "db35_rev_a302_fde8" + ], + "commands": [ + { + "skip": false, + "type": "snippet", + "key": "setup_ledger_and_config", + "variables": { + "NL_CONFIG": "nanolab/configs/10node_network/nl_config_default.toml", + "REMOTE_IP": "127.0.0.1", + "SETUP_NODES": "nl_genesis nl_pr1 nl_pr2 nl_pr3 nl_pr4 nl_pr5 nl_pr6 nl_pr7 nl_pr8 nl_pr9 nl_pr10", + "LEDGER": "nanolab/data/ledgers/10pr_init.ldb" + } + }, + { + "skip": false, + "type": "python", + "method": "publish_blocks", + "variables": { + "publish_params": { + "blocks_path": "nanolab/data/blocks/10node_100k_bintree.json", + "bps": 1000, + "subset" :{ + "start_index" : 0, + "end_index" : 1000 + } + }, + "logger_type": "rpc", + "logger_timeout": 20, + "_logger_include_peers": ["nl_pr1"] + } + } + ] +} \ No newline at end of file diff --git a/nanolab/configs/10node_network/nl_config_default.toml b/nanolab/configs/10node_network/nl_config_default.toml new file mode 100644 index 0000000..eb5be91 --- /dev/null +++ b/nanolab/configs/10node_network/nl_config_default.toml @@ -0,0 +1,86 @@ +genesis_key = "12C91837C846F875F56F67CD83040A832CFC0F131AF3DFF9E502C0D43F5D2D15" +canary_key = "FB4E458CB13508353C5B2574B82F1D1D61367F61E88707F773F068FF90050BEE" +epoch_count = 2 +burn_amount = "140282366920938463463374607431768211454" +NANO_TEST_EPOCH_1 = "0xfff0000000000000" +NANO_TEST_EPOCH_2 = "0x000000000000000f" +NANO_TEST_EPOCH_2_RECV = "0x000000000000000f" +NANO_TEST_MAGIC_NUMBER = "LC" +remote_address = "127.0.0.1" +nanolooker_enable = false +nanolooker_port = 42100 +nanolooker_node_name = "genesis" +nanolooker_mongo_port = 27017 +#promexporter_enable = true +#prom_gateway = "https://nl-exporter.bnano.info" +#prom_runid = "nanoct" + +[representatives] +node_prefix = "nl" +host_port_peer = 44100 +host_port_rpc = 45100 +host_port_ws = 47100 +config_node_path = "./nano_nodes/services/default_config-node_voting-disabled.toml" +config_rpc_path = "./nano_nodes/services/default_config-rpc.toml" +docker_tag = "" +#docker_tag = "pwo_elections-up-wip-exp-delay-221106" + +[[representatives.nodes]] +name = "pr1" +seed = "1110000000000000000000000000000000000000000000000000000000000001" +vote_weight_percent = 10 +config_node_path = "./nano_nodes/services/default_config-node.toml" + +[[representatives.nodes]] +name = "pr2" +seed = "1110000000000000000000000000000000000000000000000000000000000002" +vote_weight_percent = 10 +config_node_path = "./nano_nodes/services/default_config-node.toml" + +[[representatives.nodes]] +name = "pr3" +seed = "1110000000000000000000000000000000000000000000000000000000000003" +vote_weight_percent = 10 +config_node_path = "./nano_nodes/services/default_config-node.toml" + +[[representatives.nodes]] +name = "pr4" +seed = "1110000000000000000000000000000000000000000000000000000000000004" +vote_weight_percent = 10 +config_node_path = "./nano_nodes/services/default_config-node.toml" + +[[representatives.nodes]] +name = "pr5" +seed = "1110000000000000000000000000000000000000000000000000000000000005" +vote_weight_percent = 10 +config_node_path = "./nano_nodes/services/default_config-node.toml" + +[[representatives.nodes]] +name = "pr6" +seed = "1110000000000000000000000000000000000000000000000000000000000006" +vote_weight_percent = 10 +config_node_path = "./nano_nodes/services/default_config-node.toml" + +[[representatives.nodes]] +name = "pr7" +seed = "1110000000000000000000000000000000000000000000000000000000000007" +vote_weight_percent = 10 +config_node_path = "./nano_nodes/services/default_config-node.toml" + +[[representatives.nodes]] +name = "pr8" +seed = "1110000000000000000000000000000000000000000000000000000000000008" +vote_weight_percent = 10 +config_node_path = "./nano_nodes/services/default_config-node.toml" + +[[representatives.nodes]] +name = "pr9" +seed = "1110000000000000000000000000000000000000000000000000000000000009" +vote_weight_percent = 10 +config_node_path = "./nano_nodes/services/default_config-node.toml" + +[[representatives.nodes]] +name = "pr10" +seed = "1110000000000000000000000000000000000000000000000000000000000010" +vote_weight_percent = 10 +config_node_path = "./nano_nodes/services/default_config-node.toml" diff --git a/app/configs/3node_network/easy_setup.json b/nanolab/configs/3node_network/easy_setup.json similarity index 64% rename from app/configs/3node_network/easy_setup.json rename to nanolab/configs/3node_network/easy_setup.json index 3157a10..9b390b6 100644 --- a/app/configs/3node_network/easy_setup.json +++ b/nanolab/configs/3node_network/easy_setup.json @@ -8,25 +8,25 @@ "type": "snippet", "key": "setup_ledger_and_config", "variables": { - "NL_CONFIG": "app/configs/3node_network/nl_config_default.toml", + "NL_CONFIG": "nanolab/configs/3node_network/nl_config_default.toml", "REMOTE_IP": "127.0.0.1", "SETUP_NODES": "nl_genesis nl_pr1 nl_pr2 nl_pr3", - "LEDGER": "app/data/ledgers/3pr_init.ldb" + "LEDGER": "nanolab/data/ledgers/3pr_init.ldb" } }, { "skip": false, "type": "python", "class": "NodeInteraction", - "method": "publish_blocks_test", + "method": "publish_blocks", "variables": { - "params": { - "blocks_path": "app/data/blocks/3node_net.bintree.50k.json", + "publish_params": { + "blocks_path": "nanolab/data/blocks/3node_net.bintree.50k.json", "bps": 5000 }, "logger_type": "rpc", "logger_timeout": 180, - "logger_include_peers": ["nl_genesis","nl_pr1"] + "logger_exclude_peers": ["nl_genesis","nl_pr1"] } } ] diff --git a/nanolab/configs/3node_network/nl_config_default.toml b/nanolab/configs/3node_network/nl_config_default.toml new file mode 100644 index 0000000..30dd7c2 --- /dev/null +++ b/nanolab/configs/3node_network/nl_config_default.toml @@ -0,0 +1,44 @@ +genesis_key = "12C91837C846F875F56F67CD83040A832CFC0F131AF3DFF9E502C0D43F5D2D15" +canary_key = "FB4E458CB13508353C5B2574B82F1D1D61367F61E88707F773F068FF90050BEE" +epoch_count = 2 +burn_amount = "140282366920938463463374607431768211454" +NANO_TEST_EPOCH_1 = "0xfff0000000000000" +NANO_TEST_EPOCH_2 = "0x000000000000000f" +NANO_TEST_EPOCH_2_RECV = "0x000000000000000f" +NANO_TEST_MAGIC_NUMBER = "LC" +remote_address = "172.17.0.1" +nanolooker_enable = false +nanolooker_port = 41200 +nanolooker_node_name = "genesis" +nanolooker_mongo_port = 27017 +promexporter_enable = false +#prom_gateway = "https://nl-exporter.bnano.info" +#prom_runid = "nano-speed-testsuite" + + +[representatives] +node_prefix = "nl" +host_port_peer = 44100 +host_port_rpc = 45100 +host_port_ws = 47100 +config_node_path = "./nano_nodes/services/default_config-node_voting-disabled.toml" +config_rpc_path = "./nano_nodes/services/default_config-rpc.toml" +docker_tag = "nanocurrency/nano-beta:latest" + +[[representatives.nodes]] +name = "pr1" +seed = "1110000000000000000000000000000000000000000000000000000000000001" +vote_weight_percent = 33 +config_node_path = "./nano_nodes/services/default_config-node.toml" + +[[representatives.nodes]] +name = "pr2" +seed = "1110000000000000000000000000000000000000000000000000000000000002" +vote_weight_percent = 33 +config_node_path = "./nano_nodes/services/default_config-node.toml" + +[[representatives.nodes]] +name = "pr3" +seed = "1110000000000000000000000000000000000000000000000000000000000003" +vote_weight_percent = 33 +config_node_path = "./nano_nodes/services/default_config-node.toml" \ No newline at end of file diff --git a/nanolab/configs/nodes/default_config-node-voting_disabled.toml b/nanolab/configs/nodes/default_config-node-voting_disabled.toml new file mode 100644 index 0000000..01351fd --- /dev/null +++ b/nanolab/configs/nodes/default_config-node-voting_disabled.toml @@ -0,0 +1,19 @@ +[node.websocket] +address = "::ffff:0.0.0.0" +# Enable or disable WebSocket server. +enable = true +port = 17078 + +[rpc] +enable = true +enable_sign_hash = true + +[node] +work_threads = 1 +enable_voting = false +peering_port = 17075 +preconfigured_peers = [] +allow_local_peers = true +#bandwidth_limit = 4096 + + diff --git a/nanolab/configs/nodes/default_config-node.toml b/nanolab/configs/nodes/default_config-node.toml new file mode 100644 index 0000000..fde8578 --- /dev/null +++ b/nanolab/configs/nodes/default_config-node.toml @@ -0,0 +1,19 @@ +[node.websocket] +address = "::ffff:0.0.0.0" +# Enable or disable WebSocket server. +enable = true +port = 17078 + +[rpc] +enable = true +enable_sign_hash = true + +[node] +work_threads = 1 +enable_voting = true +peering_port = 17075 +preconfigured_peers = [] +allow_local_peers = true +#bandwidth_limit = 4096 + + diff --git a/app/data/blocks/.gitignore b/nanolab/data/blocks/.gitignore similarity index 100% rename from app/data/blocks/.gitignore rename to nanolab/data/blocks/.gitignore diff --git a/app/data/download_data.sh b/nanolab/data/download_data.sh similarity index 59% rename from app/data/download_data.sh rename to nanolab/data/download_data.sh index a36d034..af138e9 100755 --- a/app/data/download_data.sh +++ b/nanolab/data/download_data.sh @@ -1,13 +1,13 @@ #download ledgers -wget -q --continue https://bnano.info/public/saved_ledgers/3pr_init.ldb -P app/data/ledgers/ & -wget -q --continue https://bnano.info/public/saved_ledgers/6pr_init.ldb -P app/data/ledgers/ & -wget -q --continue https://bnano.info/public/saved_ledgers/10pr_init.ldb -P app/data/ledgers/ & -wget -q --continue https://bnano.info/public/saved_ledgers/6pr_bucket0-1-88-90-100_10kaccs.ldb -P app/data/ledgers/ & -wget -q --continue https://bnano.info/public/saved_ledgers/10pr_bucket0-1-88-90-100_10kaccs.ldb -P app/data/ledgers/ +wget -q --continue https://bnano.info/public/saved_ledgers/3pr_init.ldb -P nanolab/data/ledgers/ & +wget -q --continue https://bnano.info/public/saved_ledgers/6pr_init.ldb -P nanolab/data/ledgers/ & +wget -q --continue https://bnano.info/public/saved_ledgers/10pr_init.ldb -P nanolab/data/ledgers/ & +wget -q --continue https://bnano.info/public/saved_ledgers/6pr_bucket0-1-88-90-100_10kaccs.ldb -P nanolab/data/ledgers/ & +wget -q --continue https://bnano.info/public/saved_ledgers/10pr_bucket0-1-88-90-100_10kaccs.ldb -P nanolab/data/ledgers/ #download blocks -wget -q --continue https://bnano.info/public/saved_blocks/3node_net.bintree.50k.json -P app/data/blocks/ & -wget -q --continue https://bnano.info/public/saved_blocks/6node_bintree_100k.json -P app/data/blocks/ & -wget -q --continue https://bnano.info/public/saved_blocks/6node_buckets_0-1-88-90-100_10rounds.json -P app/data/blocks/ & -wget -q --continue https://bnano.info/public/saved_blocks/10node_100k_bintree.json -P app/data/blocks/ & -wget -q --continue https://bnano.info/public/saved_blocks/10node_bucket_rounds.json -P app/data/blocks/ \ No newline at end of file +wget -q --continue https://bnano.info/public/saved_blocks/3node_net.bintree.50k.json -P nanolab/data/blocks/ & +wget -q --continue https://bnano.info/public/saved_blocks/6node_bintree_100k.json -P nanolab/data/blocks/ & +wget -q --continue https://bnano.info/public/saved_blocks/6node_buckets_0-1-88-90-100_10rounds.json -P nanolab/data/blocks/ & +wget -q --continue https://bnano.info/public/saved_blocks/10node_100k_bintree.json -P nanolab/data/blocks/ & +wget -q --continue https://bnano.info/public/saved_blocks/10node_bucket_rounds.json -P nanolab/data/blocks/ \ No newline at end of file diff --git a/app/data/ledgers/.gitignore b/nanolab/data/ledgers/.gitignore similarity index 100% rename from app/data/ledgers/.gitignore rename to nanolab/data/ledgers/.gitignore diff --git a/nanolab/decorators.py b/nanolab/decorators.py new file mode 100644 index 0000000..756877f --- /dev/null +++ b/nanolab/decorators.py @@ -0,0 +1,27 @@ +# decorators.py +import asyncio +import time +from functools import wraps + + +def ensure_duration(duration=2): + + def decorator(func): + + @wraps(func) + async def wrapper(*args, **kwargs): + start_time = time.perf_counter() + result = await func(*args, **kwargs) + end_time = time.perf_counter() + + elapsed_time = end_time - start_time + remaining_time = duration - elapsed_time + + if remaining_time > 0: + await asyncio.sleep(remaining_time) + + return result + + return wrapper + + return decorator \ No newline at end of file diff --git a/run.py b/nanolab/main.py similarity index 93% rename from run.py rename to nanolab/main.py index c327905..d85b9b2 100755 --- a/run.py +++ b/nanolab/main.py @@ -7,9 +7,12 @@ from collections import defaultdict import inspect import argparse -from app import pycmd +from nanolab import pycmd +import logging -default_class = "NodeCommands" +default_class = "NodeInteraction" + +logger = logging.getLogger(__name__) def load_json(file_path): @@ -32,6 +35,7 @@ def parse_args(): def load_resolved_path(arg_value, env_var, default_path): config_path = arg_value if arg_value else os.environ.get( env_var, default_path) + logger.info("path is %s", config_path) return load_json(config_path) @@ -150,7 +154,7 @@ def execute_snippet(command_config, snippets): def execute_python(command_config): - class_name = command_config.get("class", "NodeCommands") + class_name = command_config.get("class", default_class) cls = getattr(pycmd, class_name) instance = cls(**command_config.get('constructor_params', {})) @@ -204,6 +208,12 @@ def execute_command_sequence(commands): for command_config in commands: if command_config['type'] == 'python': execute_python(command_config) + elif command_config['type'] == 'bash': + execute_bash(command_config) + else: + raise ValueError( + f"{command_config['type']} not supported for sequenced execution" + ) def execute_threaded(commands): @@ -249,7 +259,7 @@ def execute_command(command_config, snippets): def load_and_validate_configs(args): default_config_path = 'config.example.json' - default_snippets_path = 'app/snippets.json' + default_snippets_path = 'nanolab/snippets.json' config = load_resolved_path(args.config, "CONFIG_FILE", default_config_path) @@ -271,11 +281,11 @@ def execute_commands(config, snippets): execute_command(command_config, snippets) -def main(args=None): - +def main(): + args = parse_args() config, snippets = load_and_validate_configs(args) execute_commands(config, snippets) if __name__ == "__main__": - main(parse_args()) + main() diff --git a/app/node_interaction.py b/nanolab/node_interaction.py similarity index 79% rename from app/node_interaction.py rename to nanolab/node_interaction.py index 7876b24..34f99f2 100644 --- a/app/node_interaction.py +++ b/nanolab/node_interaction.py @@ -1,13 +1,14 @@ import os -os.environ["NL_APP_DIR"] = "submodules/nanolocal/nanolocal" +os.environ["NL_APP_DIR"] = "." -from submodules.xnomin.peers import get_connected_socket_endpoint, message_header, block_state, block_type_enum, message_type_enum, network_id, message_type, get_peers_from_service -from submodules.xnomin.handshake import node_handshake_id +from nanolab.xnomin.peers import get_connected_socket_endpoint, message_header, block_state, block_type_enum, message_type_enum, network_id, message_type, get_peers_from_service +from nanolab.xnomin.handshake import node_handshake_id -from nanolocal.common.nl_parse_config import ConfigParser, ConfigReadWrite -from nanolocal.common.nl_rpc import NanoRpc -from app.node_tools import StatsLogger +from nanomock.modules.nl_parse_config import ConfigParser, ConfigReadWrite +from nanomock.modules.nl_rpc import NanoRpc +from nanolab.node_tools import StatsLogger +from nanolab.decorators import ensure_duration from typing import Any, Dict, List, Optional import asyncio @@ -18,56 +19,56 @@ def load_nodes_config(): """Load nodes configuration from a file.""" - config_parser = ConfigParser() + config_parser = ConfigParser(os.environ.get("NL_APP_DIR", ".")) return config_parser.get_nodes_config() +async def create_logger(node, logger_type, hashes, logger_timeout, + logger_expected_count): + + nanorpc = NanoRpc(node["rpc_url"]) + node_version = nanorpc.version() + formatted_node_version = f'{node_version["node_vendor"]} {node_version["build_info"][0:7]}' + start_block_count = nanorpc.block_count() + + logger = StatsLogger(logger_type, + node["name"], + formatted_node_version, + hashes, + start_block_count, + timeout=logger_timeout, + expected_block_count=logger_expected_count, + ws_url=node["ws_url"], + rpc_url=node["rpc_url"]) + return logger + + +#Allow some time, to ensure StatsLoggers are correctly initialized before blocks are published by any thread +@ensure_duration(duration=2) async def create_loggers(hashes, logger_type=None, logger_timeout=None, included_peers=None, - excluded_peers=None): - """ - Create loggers for nodes. - - :param hashes: list of hashes to log - :param logger_type: type of logger to use - :param logger_timeout: logger timeout - :param included_peers: list of peers to include in logging - :param excluded_peers: list of peers to exclude from logging - :return: list of logger instances - """ + excluded_peers=None, + logger_expected_count=None): + + if not logger_type: return [] nodes_config = load_nodes_config() - loggers = [] + tasks = [] for node in nodes_config: if included_peers and node["name"] not in included_peers: continue if excluded_peers and node["name"] in excluded_peers: continue + tasks.append( + create_logger(node, logger_type, hashes, logger_timeout, + logger_expected_count)) - nanorpc = NanoRpc(node["rpc_url"]) - node_version = nanorpc.version() - formatted_node_version = f'{node_version["node_vendor"]} {node_version["build_info"][0:7]}' - start_block_count = nanorpc.block_count() - - logger = StatsLogger(logger_type, - node["name"], - formatted_node_version, - hashes, - start_block_count, - timeout=logger_timeout, - ws_url=node["ws_url"], - rpc_url=node["rpc_url"]) - loggers.append(logger) + loggers = await asyncio.gather(*tasks) return loggers async def start_loggers(loggers): - """ - Start loggers. - - :param loggers: list of logger instances - """ logger_tasks = [asyncio.create_task(logger.start()) for logger in loggers] await asyncio.gather(*logger_tasks) @@ -76,25 +77,17 @@ async def xnolib_publish(params: dict, logger_type=None, logger_timeout=None, included_peers=None, - excluded_peers=None): - """ - Publish data using Xnolib and enable logging. - - :param logger_type: type of logger to use - :param logger_timeout: logger timeout - :param included_peers: list of peers to include in logging - :param excluded_peers: list of peers to exclude from logging - """ - # params = { - # "blocks_path": "app/data/blocks/3node_net.bintree.50k.json", - # "bps": 1000 - # } + excluded_peers=None, + logger_expected_count=None): + block_lists = get_blocks_from_disk(params) sp = SocketPublish(params) messages, hashes = sp.flatten_messages(block_lists) loggers = await create_loggers(hashes, logger_type, logger_timeout, - included_peers, excluded_peers) + included_peers, excluded_peers, + logger_expected_count) + enable_logging_task = asyncio.create_task(start_loggers(loggers)) sp_task = asyncio.create_task(sp.run(messages)) @@ -103,7 +96,7 @@ async def xnolib_publish(params: dict, def read_blocks_from_disk(path, seeds=False, hashes=False, blocks=False): - res = ConfigReadWrite().read_json(path) + res = ConfigReadWrite(os.environ.get("NL_APP_DIR", ".")).read_json(path) if seeds: return res["s"] if hashes: return res["h"] if blocks: return res["b"] @@ -132,7 +125,6 @@ def get_block_subset(all_blocks: dict, start_round: int, end_round: int, x[start_index:end_index] for x in all_blocks['h'][start_round:end_round] ] - return blocks @@ -193,7 +185,8 @@ def __str__(self): return str(self.hdr) + "\n" + str(self.block) def get_xnolib_context(self, peers=None): - ctx = ConfigParser().get_xnolib_localctx() + ctx = ConfigParser(os.environ.get("NL_APP_DIR", + ".")).get_xnolib_localctx() ctx["net_id"] = network_id(ord('X')) if peers is not None: # chose a single peer , if enabled in config file @@ -301,4 +294,9 @@ def create_default_tasks(self, sockets, messages): async def run(self, messages: List[Any]) -> int: tasks = self.create_publish_tasks(self.sockets, messages) await asyncio.gather(*tasks) - return len(messages) + # make sure the last few blocks are published. + #ideally this would check if all messages are received + await asyncio.sleep(15) + + message_count = len(messages) + return message_count diff --git a/app/node_tools.py b/nanolab/node_tools.py similarity index 58% rename from app/node_tools.py rename to nanolab/node_tools.py index 715a38c..5249de5 100644 --- a/app/node_tools.py +++ b/nanolab/node_tools.py @@ -1,11 +1,11 @@ import json import ssl import threading -#import websocket +import websocket import asyncio import time from typing import List, Any -from nanolocal.common.nl_rpc import NanoRpc +from nanomock.modules.nl_rpc import NanoRpc class StatsLogger: @@ -17,8 +17,10 @@ def __init__(self, hashes: List[Any], start_block_count: dict, timeout: int = 600, + expected_block_count=None, ws_url: str = None, rpc_url: str = None): + self.logger_type = logger_type self.timeout = timeout self.node_name = node_name @@ -26,19 +28,40 @@ def __init__(self, self.ws_url = ws_url self.rpc_url = rpc_url self.hashes = hashes - self.start_block_count = start_block_count - self.expected_blocks_count = len(hashes) - self.synced_block_count = self.expected_blocks_count + int( - start_block_count["count"]) + #self.start_block_count = start_block_count + self.count_start = int(start_block_count["count"]) + self.cemented_start = int(start_block_count["cemented"]) + self.expected_blocks_count = expected_block_count or len(hashes) + self.count_total = self.expected_blocks_count + self.count_start self.nanorpc = NanoRpc(self.rpc_url) + self.logging_task = None def is_fully_synced(self): block_count = self.nanorpc.block_count() - cemented_diff = int(block_count["cemented"]) - int( - self.start_block_count["cemented"]) + + cemented = int(block_count["cemented"]) + count = int(block_count["count"]) + + cemented_diff = cemented - self.cemented_start is_synced = cemented_diff == self.expected_blocks_count - return is_synced, int(block_count["count"]), int( - block_count["cemented"]) + return is_synced, count, cemented + + def print_rpc_stats(self, elapsed_time, check_count, cemented_count, + previous_count, previous_cemented): + bps = 0 if previous_count == 0 else check_count - previous_count + cps = 0 if previous_cemented == 0 else cemented_count - previous_cemented + print( + f"{elapsed_time:>4} seconds {self.node_name:>12} | {self.node_version} | \ +{cemented_count:>7}/{check_count:>7}/{self.count_total:>7} @{bps:>5} bps \ +(avg: {round((check_count - self.count_start)/max(1, elapsed_time),2)}) | \ +@{cps:>5} cps (avg: {round((cemented_count - self.cemented_start)/max(1, elapsed_time),2)})" + ) + + def print_ws_stats(self, elapsed_time, cemented_count, previous_cemented): + cps = 0 if previous_cemented == 0 else cemented_count - previous_cemented + print( + f"""{elapsed_time:>4} seconds {self.node_name:>12} | {self.node_version} | @{cps:>5} cps (avg: {round((cemented_count)/max(1, elapsed_time),2)})""" + ) async def log_rpc(self): if not self.rpc_url: @@ -48,43 +71,50 @@ async def log_rpc(self): previous_cemented = 0 start_time = time.time() - for _ in range(self.timeout): + while True: is_synced, check_count, cemented_count = self.is_fully_synced() - elapsed_time = int(time.time() - start_time) - bps = check_count - previous_count - cps = cemented_count - previous_cemented + self.print_rpc_stats(elapsed_time, check_count, cemented_count, + previous_count, previous_cemented) - print( - f"{elapsed_time:>4} seconds {self.node_name:>12} - {self.node_version} - {cemented_count:>7}/{check_count:>7}/{self.synced_block_count:>7} - @{bps:>5} bps - @{cps:>5} cps" - ) - - if is_synced: + if is_synced or elapsed_time > self.timeout: break previous_count = check_count previous_cemented = cemented_count await asyncio.sleep(1) - async def log_websocket(self, ws_url): - if not self.ws_url: raise ValueError("ws_url must be defined") - # Initialize the SimpleWs class with the required parameters - simple_ws = SimpleWs(ws_url, + async def log_websocket(self): + if not self.ws_url: + raise ValueError("ws_url must be defined") + + simple_ws = SimpleWs(self.ws_url, self.node_name, self.node_version, ws_topics=["confirmation"]) - # Implement your logic to monitor the WebSocket messages and calculate the stats - # You may need to use additional synchronization mechanisms like asyncio.Lock or asyncio.Queue - # to safely access the shared data between the StatsLogger and SimpleWs classes + cemented_count = 0 + previous_cemented = 0 + start_time = time.time() + + while cemented_count < len(self.hashes): + cemented_count = simple_ws.confirmed_hashes + elapsed_time = int(time.time() - start_time) + self.print_ws_stats(elapsed_time, cemented_count, + previous_cemented) + + if elapsed_time > self.timeout: + break + + previous_cemented = cemented_count + await asyncio.sleep(1) - # When finished, close the WebSocket connection simple_ws.ws.close() async def start(self): if self.logger_type == "rpc": await self.log_rpc() - elif self.logger_type == "websocket": + elif self.logger_type == "ws": await self.log_websocket() else: raise ValueError( @@ -115,6 +145,8 @@ def __init__(self, self.ws_url = ws_url self.node_name = node_name self.ws_topics = ws_topics + self.confirmed_hashes = 0 + self.lock = threading.Lock() self.ws = websocket.WebSocketApp( ws_url, @@ -136,8 +168,12 @@ def __init__(self, self.wst.start() + def increment_confirmed_hash_count(self): + with self.lock: + self.confirmed_hashes += 1 + def on_message(self, ws, message): - print(message) + self.increment_confirmed_hash_count() # You can implement your logic for handling WebSocket messages here. def on_error(self, ws, error): diff --git a/nanolab/pycmd.py b/nanolab/pycmd.py new file mode 100644 index 0000000..b7fbd4b --- /dev/null +++ b/nanolab/pycmd.py @@ -0,0 +1,34 @@ +import time +import nanolab.node_interaction as nni +import asyncio + + +class TestClass: + + def long_running_method(self, duration=1): + time.sleep(duration) + + +class NodeInteraction: + + def __init__(self): + # Your existing code... + self.logger = None + + def publish_blocks(self, + publish_params, + logger_type=None, + logger_timeout=600, + logger_include_peers=None, + logger_exclude_peers=None, + logger_expected_count=None): + ''' + mandatory publish_params: blocks_path, bps + optional publish_params: peers, split, split_skip, reverse, shuffle, + start_round, end_round, subset.start_index, subset.end_index + ''' + + asyncio.run( + nni.xnolib_publish(publish_params, logger_type, logger_timeout, + logger_include_peers, logger_exclude_peers, + logger_expected_count)) diff --git a/app/snippets.json b/nanolab/snippets.json similarity index 55% rename from app/snippets.json rename to nanolab/snippets.json index e045802..753ef0c 100644 --- a/app/snippets.json +++ b/nanolab/snippets.json @@ -9,31 +9,31 @@ "commands": [ { "type": "bash", - "command": "./nl_run down " + "command": "nanomock down " }, { "type": "bash", - "command": "cp {NL_CONFIG} submodules/nanolocal/nanolocal/nl_config.toml" + "command": "cp {NL_CONFIG} ./nl_config.toml" }, { "type": "bash", - "command": "./nl_run conf_edit --nested_path remote_address --value {REMOTE_IP}" + "command": "nanomock conf_edit --nested_path remote_address --value {REMOTE_IP}" }, { "type": "bash", - "command": "./nl_run conf_edit --nested_path representatives.docker_tag --value {docker_tag}" + "command": "#nanomock conf_edit --nested_path representatives.docker_tag --value {docker_tag}" }, { "type": "bash", - "command": "./nl_run create && ./nl_run reset" + "command": "nanomock create && nanomock reset" }, { "type": "bash", - "command": "for i in {SETUP_NODES}; do cp {LEDGER} submodules/nanolocal/nanolocal/nano_nodes/$i/NanoTest/data.ldb; done" + "command": "for i in {SETUP_NODES}; do cp {LEDGER} ./nano_nodes/$i/NanoTest/data.ldb; done" }, { "type": "bash", - "command": "./nl_run start && ./nl_run init_wallets" + "command": "nanomock start && nanomock init_wallets" } ] }, @@ -46,23 +46,23 @@ "commands": [ { "type": "bash", - "command": "./nl_run down" + "command": "nanomock down" }, { "type": "bash", - "command": "cp {NL_CONFIG} submodules/nanolocal/nanolocal/nl_config.toml" + "command": "cp {NL_CONFIG} ./nl_config.toml" }, { "type": "bash", - "command": "./nl_run conf_edit --nested_path remote_address --value {REMOTE_IP}" + "command": "#nanomock conf_edit --nested_path remote_address --value {REMOTE_IP}" }, { "type": "bash", - "command": "./nl_run conf_edit --nested_path representatives.docker_tag --value {docker_tag}" + "command": "#nanomock conf_edit --nested_path representatives.docker_tag --value {docker_tag}" }, { "type": "bash", - "command": "./nl_run create && ./nl_run reset && ./nl_run start && ./nl_run init && ./nl_run restart_wait_sync" + "command": "nanomock create && nanomock reset && nanomock start && nanomock init && nanomock restart_wait_sync" } ] } diff --git a/nanolab/xnomin/__init__.py b/nanolab/xnomin/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/submodules/xnomin/acctools.py b/nanolab/xnomin/acctools.py similarity index 100% rename from submodules/xnomin/acctools.py rename to nanolab/xnomin/acctools.py diff --git a/submodules/xnomin/handshake.py b/nanolab/xnomin/handshake.py similarity index 98% rename from submodules/xnomin/handshake.py rename to nanolab/xnomin/handshake.py index 5f8426c..0970017 100644 --- a/submodules/xnomin/handshake.py +++ b/nanolab/xnomin/handshake.py @@ -1,4 +1,4 @@ -from submodules.xnomin.peers import message_header, ed25519_blake2b, socket, hexlify, message_type +from nanolab.xnomin.peers import message_header, ed25519_blake2b, socket, hexlify, message_type from secrets import token_bytes from typing import Tuple diff --git a/submodules/xnomin/peers.py b/nanolab/xnomin/peers.py similarity index 99% rename from submodules/xnomin/peers.py rename to nanolab/xnomin/peers.py index e287fe3..af8c9fa 100644 --- a/submodules/xnomin/peers.py +++ b/nanolab/xnomin/peers.py @@ -8,7 +8,7 @@ import ed25519_blake2b import json import requests -from submodules.xnomin.acctools import to_account_addr, account_key +from nanolab.xnomin.acctools import to_account_addr, account_key def get_peers_from_service(ctx: dict, url=None): @@ -559,7 +559,7 @@ def parse_peer(cls, data: bytes): @classmethod def from_json(self, json_peer): - from submodules.xnomin.telemetry_req import telemetry_ack + from nanolab.xnomin.telemetry_req import telemetry_ack # Add 'incoming' argument when peer service code gets updated peer = Peer(ip_addr(json_peer['ip']), json_peer['port'], json_peer['score'], json_peer['is_voting']) #, diff --git a/submodules/xnomin/telemetry_req.py b/nanolab/xnomin/telemetry_req.py similarity index 96% rename from submodules/xnomin/telemetry_req.py rename to nanolab/xnomin/telemetry_req.py index ddaf8ef..cbabfdf 100755 --- a/submodules/xnomin/telemetry_req.py +++ b/nanolab/xnomin/telemetry_req.py @@ -3,7 +3,7 @@ import struct import binascii -from submodules.xnomin.peers import message_header, message_type, message_type_enum +from nanolab.xnomin.peers import message_header, message_type, message_type_enum class telemetry_req: diff --git a/nl_run b/nl_run deleted file mode 100755 index 69b0d47..0000000 --- a/nl_run +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash -cd submodules/nanolocal -./nl_run.py "$@" -cd - > /dev/null \ No newline at end of file diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..13bd4d0 --- /dev/null +++ b/setup.py @@ -0,0 +1,22 @@ +from setuptools import setup, find_packages + +with open("README.md", "r") as fh: + long_description = fh.read() + +setup(name="nanolab", + version="0.0.1", + author="gr0vity", + description="testing tool using nanomock", + long_description=long_description, + long_description_content_type="text/markdown", + url="https://github.com/gr0vity-dev/nanolab", + packages=find_packages(exclude=["unit_tests"]), + include_package_data=True, + install_requires=[ + "nanomock==0.0.6", + ], + entry_points={ + 'console_scripts': [ + 'nanolab=nanolab.main:main', + ], + }) diff --git a/setup_venv.sh b/setup_venv.sh index 1c1c155..3dc8e68 100755 --- a/setup_venv.sh +++ b/setup_venv.sh @@ -25,7 +25,7 @@ create_venv() { git submodule update --init --recursive echo "Downloading blocks and ledgers..." - app/data/download_data.sh + nanolab/data/download_data.sh echo "Setup complete." } diff --git a/submodules/nanolocal b/submodules/nanolocal deleted file mode 160000 index 1e9319e..0000000 --- a/submodules/nanolocal +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 1e9319ef9bc09ca78521c7494defb78b014cfc0a diff --git a/unit_tests/test_configs/invalid_method.json b/unit_tests/test_configs/invalid_method.json index 2d4145c..21c0125 100644 --- a/unit_tests/test_configs/invalid_method.json +++ b/unit_tests/test_configs/invalid_method.json @@ -3,7 +3,7 @@ "commands": [ { "type": "python", - "class": "NodeCommands", + "class": "NodeInteraction", "method": "non_existent_method" } ] diff --git a/unit_tests/test_configs/nanolocal_down.json b/unit_tests/test_configs/nanolocal_down.json index a41edb6..b7bd50e 100644 --- a/unit_tests/test_configs/nanolocal_down.json +++ b/unit_tests/test_configs/nanolocal_down.json @@ -3,7 +3,7 @@ "commands": [ { "type": "bash", - "command": "./nl_run down" + "command": "nanomock down" } ] } diff --git a/unit_tests/test_run.py b/unit_tests/test_run.py index fb66545..3b2d96a 100644 --- a/unit_tests/test_run.py +++ b/unit_tests/test_run.py @@ -5,12 +5,12 @@ import io from pathlib import Path from unittest.mock import patch -import run +import nanolab.main as run import pytest from argparse import Namespace -#@patch decorator is used to replace 'app.run.NodeCommands' with the following TestClass +#@patch decorator is used to replace 'app.run.NodeInteraction' with the following TestClass class TestClass: def long_running_method(self, duration=1): @@ -57,13 +57,11 @@ def contains_expected_substring(self, expected_substring): self.assertIn(expected_substring, output) - #@patch('app.pycmd.NodeCommands', new=TestClass) def test_invalid_type(self): self.load_config_to_env('unit_tests/test_configs/invalid_type.json') expected_message = "Invalid command type 'invalid' at index 0 in 'commands'" self.match_expected_error(ValueError, expected_message) - #@patch('app.pycmd.NodeCommands', new=TestClass) def test_invalid_snippet_key(self): self.load_config_to_env( 'unit_tests/test_configs/invalid_snippet_key.json') @@ -72,7 +70,7 @@ def test_invalid_snippet_key(self): def test_invalid_method(self): self.load_config_to_env('unit_tests/test_configs/invalid_method.json') - expected_message = "Python command 0: Method 'non_existent_method' not found in class 'NodeCommands'." + expected_message = "Python command 0: Method 'non_existent_method' not found in class 'NodeInteraction'." self.match_expected_error(ValueError, expected_message) def test_invalid_snippet_missing_mandatory_var(self): @@ -82,7 +80,6 @@ def test_invalid_snippet_missing_mandatory_var(self): expected_message = "'Missing value(s) for mandatory variable(s): unused_mandatory_var'" self.match_expected_error(KeyError, expected_message) - #@patch('app.pycmd.NodeCommands', new=TestClass) def test_threaded_parallel(self): start_time = time.time() self.load_config_to_env(