Refactor Command execution. Many smaller bugfixes

MANIFEST.in

@@ -0,0 +1 @@
+recursive-include nanolab/snippets/ *

nanolab/command/command.py

@@ -0,0 +1,49 @@
+from abc import ABC, abstractmethod
+from .mixins.bash_command_mixin import BashCommandMixin
+from .mixins.snippet_command_mixin import SnippetCommandMixin
+from .mixins.python_command_mixin import PythonCommandMixin
+from .mixins.threaded_command_mixin import ThreadedCommandMixin
+from nanolab.src.snippet_manager import SnippetManager
+
+
+class ICommand(ABC):
+
+    @abstractmethod
+    def validate(self):
+        pass
+
+    @abstractmethod
+    def execute(self):
+        pass
+
+
+class Command:
+
+    def __init__(self, command_config: dict, snippet_manager: SnippetManager):
+        self.command_config = command_config
+        self.snippet_manager = snippet_manager
+        self.mixins = {
+            'bash': BashCommandMixin,
+            'snippet': SnippetCommandMixin,
+            'python': PythonCommandMixin,
+            'threaded': ThreadedCommandMixin
+        }
+
+        command_type = self.command_config['type']
+
+        if command_type not in self.mixins:
+            raise ValueError(f"Unsupported command type '{command_type}'")
+
+        mixin = self.mixins[command_type](self)
+        self.mixin = mixin
+
+    def validate(self):
+        self.mixin.validate()
+
+    def execute(self):
+        self.mixin.execute()
+
+    def execute_another_command(self, command_config):
+        another_command = Command(command_config, self.snippet_manager)
+        another_command.validate()
+        another_command.execute()

nanolab/command/command_validator.py

@@ -0,0 +1,9 @@
+from .command import Command
+
+
+class CommandValidator:
+
+    @staticmethod
+    def validate_command(command_config: dict, snippets: dict = None):
+        command = Command(command_config, snippets)
+        command.validate()

nanolab/command/mixins/base_mixin.py

@@ -0,0 +1,29 @@
+from os import environ
+import inspect
+import logging
+
+
+class CommandMixinBase:
+
+    def __init__(self, command_instance):
+        self.command_instance = command_instance
+        self.logger = logging.getLogger(__name__)
+
+    def _get_completed_variables(self):
+        variables = self.command_instance.command_config.get('variables', {})
+
+        #add global variables
+        variables["docker_tag"] = environ["docker_tag"]
+        return variables
+
+    def _get_filtered_variables(self, method):
+        variables = self._get_completed_variables()
+        method_parameters = self._get_method_parameters(method)
+        return {
+            key: value
+            for key, value in variables.items() if key in method_parameters
+        }
+
+    def _get_method_parameters(self, method):
+        signature = inspect.signature(method)
+        return [param.name for param in signature.parameters.values()]

nanolab/command/mixins/bash_command_mixin.py

@@ -0,0 +1,26 @@
+import subprocess
+from .base_mixin import CommandMixinBase
+
+
+class BashCommandMixin(CommandMixinBase):
+
+    def validate(self):
+        pass
+        # Validation logic for BashCommand
+
+    def execute(self):
+        variables = self._get_completed_variables()
+        command = self.command_instance.command_config['command'].format(
+            **variables)
+        process = subprocess.run(command,
+                                 shell=True,
+                                 text=True,
+                                 capture_output=True)
+
+        if process.returncode != 0:
+            print(
+                f"Error executing command: {command}\nError: {process.stderr.strip()}"
+            )
+        else:
+            response = process.stdout.strip() or process.stderr.strip()
+            print(response)

nanolab/command/mixins/python_command_mixin.py

@@ -0,0 +1,40 @@
+from .base_mixin import CommandMixinBase
+from nanolab import pycmd
+
+
+class PythonCommandMixin(CommandMixinBase):
+
+    def validate(self):
+
+        if "method" not in self.command_instance.command_config:
+            raise ValueError("Python command: 'method' is required.")
+
+        method_name = self.command_instance.command_config["method"]
+
+        if "class" in self.command_instance.command_config:
+            cls = getattr(pycmd, self.command_instance.command_config["class"])
+            if not cls:
+                raise ValueError(
+                    f"Python command: Class '{self.command_instance.command_config['class']}' not found."
+                )
+            if not hasattr(cls, method_name):
+                raise ValueError(
+                    f"Python command: Method '{method_name}' not found in class '{self.command_instance.command_config['class']}'."
+                )
+        else:
+            if method_name not in globals():
+                raise ValueError(
+                    f"Python command: Method '{method_name}' not found in the current module."
+                )
+
+    def execute(self):
+        class_name = self.command_instance.command_config.get(
+            "class", "NodeInteraction")
+        cls = getattr(pycmd, class_name)
+        instance = cls(**self.command_instance.command_config.get(
+            'constructor_params', {}))
+
+        method = getattr(instance,
+                         self.command_instance.command_config['method'])
+        filtered_variables = self._get_filtered_variables(method)
+        method(**filtered_variables)

nanolab/command/mixins/snippet_command_mixin.py

@@ -0,0 +1,23 @@
+from nanolab.src.snippet_manager import SnippetManager
+from .base_mixin import CommandMixinBase
+
+
+class SnippetCommandMixin(CommandMixinBase):
+
+    def validate(self):
+        snippet = self.command_instance.snippet_manager.get_snippet_by_key(
+            self.command_instance.command_config["key"])
+        missing_vars = set(snippet.get("mandatory_vars", set())) - set(
+            self.command_instance.command_config.get("variables", {}).keys())
+        if missing_vars:
+            raise KeyError(
+                f"Missing value(s) for mandatory variable(s): {', '.join(missing_vars)}"
+            )
+
+    def execute(self):
+        snippet = self.command_instance.snippet_manager.get_snippet_by_key(
+            self.command_instance.command_config["key"])
+
+        for snippet_command in snippet["commands"]:
+            snippet_command["variables"] = self._get_completed_variables()
+            self.command_instance.execute_another_command(snippet_command)

nanolab/command/mixins/threaded_command_mixin.py

@@ -0,0 +1,61 @@
+import threading
+from nanolab.src.snippet_manager import SnippetManager
+from nanolab import pycmd
+from .base_mixin import CommandMixinBase
+from collections import defaultdict
+
+
+class ThreadedCommandMixin(CommandMixinBase):
+
+    def _execute_command_sequence(self, commands):
+        for command_config in commands:
+            self.command_instance.execute_another_command(command_config)
+
+    def validate(self):
+        for python_command in self.command_instance.command_config["commands"]:
+            if "method" not in python_command:
+                raise ValueError(f"Python command: 'method' is required.")
+
+            method_name = python_command["method"]
+
+            if "class" in python_command:
+                cls = getattr(pycmd, python_command["class"])
+                if not cls:
+                    raise ValueError(
+                        f"Python command: Class '{python_command['class']}' not found."
+                    )
+                if not hasattr(cls, method_name):
+                    raise ValueError(
+                        f"Python command: Method '{method_name}' not found in class '{python_command['class']}'."
+                    )
+            else:
+                if method_name not in globals():
+                    raise ValueError(
+                        f"Python command: Method '{method_name}' not found in the current module."
+                    )
+
+    def execute(self):
+        threads = []
+        command_groups = defaultdict(list)
+
+        for command_config in self.command_instance.command_config["commands"]:
+            group = command_config.get("group")
+            if group:
+                command_groups[group].append(command_config)
+            else:
+                thread = threading.Thread(
+                    target=self.command_instance.execute_another_command,
+                    args=(command_config, ))
+                threads.append(thread)
+
+        #execute these commands in sequence in one thread
+        for group_commands in command_groups.values():
+            thread = threading.Thread(target=self._execute_command_sequence,
+                                      args=(group_commands, ))
+            threads.append(thread)
+
+        for thread in threads:
+            thread.start()
+
+        for thread in threads:
+            thread.join()

nanolab/data/download_data.sh

@@ -1,13 +1,13 @@
 #download ledgers 
-wget -q --continue https://bnano.info/public/saved_ledgers/3pr_init.ldb -P nanolab/data/ledgers/ &
-wget -q --continue https://bnano.info/public/saved_ledgers/6pr_init.ldb -P nanolab/data/ledgers/ &
-wget -q --continue https://bnano.info/public/saved_ledgers/10pr_init.ldb -P nanolab/data/ledgers/ &
-wget -q --continue https://bnano.info/public/saved_ledgers/6pr_bucket0-1-88-90-100_10kaccs.ldb -P nanolab/data/ledgers/ &
-wget -q --continue https://bnano.info/public/saved_ledgers/10pr_bucket0-1-88-90-100_10kaccs.ldb -P nanolab/data/ledgers/
+wget -q --continue https://bnano.info/public/saved_ledgers/3pr_init.ldb -P ledgers/ &
+wget -q --continue https://bnano.info/public/saved_ledgers/6pr_init.ldb -P ledgers/ &
+wget -q --continue https://bnano.info/public/saved_ledgers/10pr_init.ldb -P ledgers/ &
+wget -q --continue https://bnano.info/public/saved_ledgers/6pr_bucket0-1-88-90-100_10kaccs.ldb -P ledgers/ &
+wget -q --continue https://bnano.info/public/saved_ledgers/10pr_bucket0-1-88-90-100_10kaccs.ldb -P ledgers/
 
 #download blocks
-wget -q --continue https://bnano.info/public/saved_blocks/3node_net.bintree.50k.json -P  nanolab/data/blocks/ &
-wget -q --continue https://bnano.info/public/saved_blocks/6node_bintree_100k.json -P nanolab/data/blocks/ &
-wget -q --continue https://bnano.info/public/saved_blocks/6node_buckets_0-1-88-90-100_10rounds.json -P nanolab/data/blocks/ &
-wget -q --continue https://bnano.info/public/saved_blocks/10node_100k_bintree.json -P nanolab/data/blocks/ &
-wget -q --continue https://bnano.info/public/saved_blocks/10node_bucket_rounds.json -P nanolab/data/blocks/
+wget -q --continue https://bnano.info/public/saved_blocks/3node_net.bintree.50k.json -P  blocks/ &
+wget -q --continue https://bnano.info/public/saved_blocks/6node_bintree_100k.json -P blocks/ &
+wget -q --continue https://bnano.info/public/saved_blocks/6node_buckets_0-1-88-90-100_10rounds.json -P blocks/ &
+wget -q --continue https://bnano.info/public/saved_blocks/10node_100k_bintree.json -P blocks/ &
+wget -q --continue https://bnano.info/public/saved_blocks/10node_bucket_rounds.json -P blocks/