Dynamic repository

pyproject.toml

@@ -21,7 +21,7 @@ classifiers = [
 [project.optional-dependencies]
 c-language = ['clang']
 plots = ['matplotlib']
-tests = ['pytest', 'pytest-cov', 'pytest-mock']
+tests = ['pytest', 'pytest-cov', 'pytest-mock', 'pytest-subprocess>=1.5.3']
 checks = ['flake8>=5.0.4', 'mypy']
 docs = ['sphinx',
         'pydata-sphinx-theme>=0.13.3',

run_configs/build_all.py

@@ -21,17 +21,17 @@ class Script(Tool):
     :name: the path to the script to run.
     '''
     def __init__(self, name: Path):
-        super().__init__(name=name.name, exec_name=name,
+        super().__init__(name=name.name, executable=name,
                          category=Category.MISC)
 
     def check_available(self):
         '''Since there typically is no command line option we could test for
         the tolls here, we use `which` to determine if a tool is available.
         '''
-        out = shutil.which(self.exec_name)
+        out = shutil.which(self.executable)
         if out:
             return True
-        print(f"Tool '{self.name}' (f{self.exec_name}) cannot be executed.")
+        print(f"Tool '{self.name}' (f{self.executable}) cannot be executed.")
         return False
 
 

run_configs/lfric/lfric_common.py

@@ -20,7 +20,7 @@ class Script(Tool):
     :name: the path to the script to run.
     '''
     def __init__(self, name: Path):
-        super().__init__(name=name.name, exec_name=str(name),
+        super().__init__(name=name.name, executable=str(name),
                          category=Category.MISC)
 
     def check_available(self):

source/fab/artefacts.py

@@ -82,11 +82,11 @@ def add(self, collection: Union[str, ArtefactSet],
         self[collection].update(files)
 
     def update_dict(self, collection: Union[str, ArtefactSet],
-                    key: str, values: Union[str, Iterable]):
+                    key: Optional[str], values: Union[str, Iterable]):
         '''For ArtefactSets that are a dictionary of sets: update
         the set with the specified values.
         :param collection: the name of the collection to add this to.
-        :param key: the key in the dictionary to update.
+        :param key: None key indicates no association with an executable, a library.
         :param values: the values to update with.
         '''
         self[collection][key].update([values] if isinstance(values, str)

source/fab/build_config.py

@@ -24,8 +24,8 @@
 from fab.constants import BUILD_OUTPUT, SOURCE_ROOT, PREBUILD
 from fab.metrics import (send_metric, init_metrics, stop_metrics,
                          metrics_summary)
-from fab.tools.category import Category
-from fab.tools.tool_box import ToolBox
+from fab.category import Category
+from fab.tool_box import ToolBox
 from fab.steps.cleanup_prebuilds import CLEANUP_COUNT, cleanup_prebuilds
 from fab.util import TimerLogger, by_type, get_fab_workspace
 

source/fab/cli.py

@@ -3,16 +3,16 @@
 #  For further details please refer to the file COPYRIGHT
 #  which you should have received as part of this distribution
 # ##############################################################################
-
-'''Functions to run Fab from the command line.
-'''
-
+"""
+Command line entry point.
+"""
 import sys
 from pathlib import Path
 from typing import Dict, Optional
 
 from fab.artefacts import ArtefactSet, CollectionGetter
 from fab.build_config import BuildConfig
+from fab.category import Category
 from fab.steps.analyse import analyse
 from fab.steps.c_pragma_injector import c_pragma_injector
 from fab.steps.compile_c import compile_c
@@ -22,7 +22,8 @@
 from fab.steps.find_source_files import find_source_files
 from fab.steps.grab.folder import grab_folder
 from fab.steps.preprocess import preprocess_c, preprocess_fortran
-from fab.tools import Category, ToolBox, ToolRepository
+from fab.tool_box import ToolBox
+from fab.tool_repository import ToolRepository
 from fab.util import common_arg_parser
 
 

source/fab/parse/c.py

@@ -3,7 +3,6 @@
 # which you should have received as part of this distribution
 """
 C language handling classes.
-
 """
 import logging
 import warnings

source/fab/steps/archive_objects.py

@@ -16,7 +16,8 @@
 from fab.build_config import BuildConfig
 from fab.steps import step
 from fab.util import log_or_dot
-from fab.tools import Ar, Category
+from fab.category import Category
+from fab.tools.ar import Ar
 from fab.artefacts import ArtefactsGetter, CollectionGetter
 
 logger = logging.getLogger(__name__)

source/fab/steps/compile_c.py

@@ -19,7 +19,9 @@
 from fab.metrics import send_metric
 from fab.parse.c import AnalysedC
 from fab.steps import check_for_errors, run_mp, step
-from fab.tools import Category, Compiler, Flags
+from fab.category import Category
+from fab.flags import Flags
+from fab.tools.compiler import Compiler
 from fab.util import CompiledFile, log_or_dot, Timer, by_type
 
 logger = logging.getLogger(__name__)
@@ -133,7 +135,7 @@ def _compile_file(arg: Tuple[AnalysedC, MpCommonArgs]):
     with Timer() as timer:
         flags = Flags(mp_payload.flags.flags_for_path(path=analysed_file.fpath,
                                                       config=config))
-        obj_combo_hash = _get_obj_combo_hash(compiler, analysed_file, flags)
+        obj_combo_hash = get_obj_combo_hash(compiler, analysed_file, flags)
 
         obj_file_prebuild = (config.prebuild_folder /
                              f'{analysed_file.fpath.stem}.'
@@ -162,7 +164,7 @@ def _compile_file(arg: Tuple[AnalysedC, MpCommonArgs]):
                         output_fpath=obj_file_prebuild)
 
 
-def _get_obj_combo_hash(compiler, analysed_file, flags: Flags):
+def get_obj_combo_hash(compiler, analysed_file, flags: Flags):
     # get a combo hash of things which matter to the object file we define
     try:
         obj_combo_hash = sum([

source/fab/steps/compile_fortran.py

@@ -22,7 +22,9 @@
 from fab.metrics import send_metric
 from fab.parse.fortran import AnalysedFortran
 from fab.steps import check_for_errors, run_mp, step
-from fab.tools import Category, Compiler, Flags
+from fab.category import Category
+from fab.flags import Flags
+from fab.tools.compiler import Compiler
 from fab.util import (CompiledFile, log_or_dot_finish, log_or_dot, Timer,
                       by_type, file_checksum)
 

source/fab/steps/grab/fcm.py

@@ -8,15 +8,14 @@
 decorated with @steps since all functions here just call the
 corresponding svn steps.
 '''
-
-from typing import Optional, Union
+from typing import Optional
 
 from fab.steps.grab.svn import svn_export, svn_checkout, svn_merge
 from fab.tools import Category
 
 
 def fcm_export(config, src: str, dst_label: Optional[str] = None,
-               revision: Optional[Union[int, str]] = None):
+               revision: Optional[int] = None):
     """
     Params as per :func:`~fab.steps.grab.svn.svn_export`.
 

source/fab/steps/grab/svn.py

@@ -14,10 +14,11 @@
 import xml.etree.ElementTree as ET
 
 from fab.steps import step
-from fab.tools import Category, Versioning
+from fab.category import Category
+from fab.tools.versioning import Versioning
 
 
-def _get_revision(src, revision=None) -> Tuple[str, Union[str, None]]:
+def _get_revision(src, revision: Optional[str] = None) -> Tuple[str, Optional[str]]:
     """
     Pull out the revision if it's part of the url.
 
@@ -60,7 +61,7 @@ def _svn_prep_common(config, src: str,
 @step
 def svn_export(config, src: str,
                dst_label: Optional[str] = None,
-               revision=None,
+               revision: Optional[str] = None,
                category=Category.SUBVERSION):
     # todo: params in docstrings
     """
@@ -113,6 +114,8 @@ def check_conflict(tool: Versioning, dst: Union[str, Path]):
     '''Check if there's a conflict
     '''
     xml_str = tool.run(['status', '--xml'], cwd=dst, capture_output=True)
+    if xml_str is None:
+        raise RuntimeError("Subversion did not provide a status report")
     root = ET.fromstring(xml_str)
 
     for target in root:

source/fab/steps/preprocess.py

@@ -18,7 +18,8 @@
 from fab.build_config import BuildConfig, FlagsConfig
 from fab.metrics import send_metric
 from fab.steps import check_for_errors, run_mp, step
-from fab.tools import Category, Cpp, CppFortran, Preprocessor
+from fab.category import Category
+from fab.tools.preprocessor import Cpp, CppFortran, Preprocessor
 from fab.util import (log_or_dot_finish, input_to_output_fpath, log_or_dot,
                       suffix_filter, Timer, by_type)
 

source/fab/steps/psyclone.py

@@ -24,7 +24,8 @@
 from fab.parse.x90 import X90Analyser, AnalysedX90
 from fab.steps import run_mp, check_for_errors, step
 from fab.steps.preprocess import pre_processor
-from fab.tools import Category, Psyclone
+from fab.category import Category
+from fab.tools.psyclone import Psyclone
 from fab.util import (log_or_dot, input_to_output_fpath, file_checksum,
                       file_walk, TimerLogger, string_checksum, suffix_filter,
                       by_type, log_or_dot_finish)

source/fab/tools/tool_box.py → source/fab/tool_box.py

@@ -10,9 +10,9 @@
 import warnings
 from typing import Dict, Optional
 
-from fab.tools.category import Category
-from fab.tools.tool import Tool
-from fab.tools.tool_repository import ToolRepository
+from fab.category import Category
+from fab.tools import Tool
+from fab.tool_repository import ToolRepository
 
 
 class ToolBox:

source/fab/tools/tool_repository.py → source/fab/tool_repository.py

@@ -3,80 +3,72 @@
 # For further details please refer to the file COPYRIGHT
 # which you should have received as part of this distribution
 ##############################################################################
-
-'''This file contains the ToolRepository class.
-'''
-
-# We can't declare _singleton and __new__() using ToolRepository, but
-# it is allowed if we use this import:
+"""
+Collection of all tools known to Fab.
+"""
 from __future__ import annotations
 
-import logging
-from typing import cast, Optional
+from typing import cast, Optional, Type
 
-from fab.tools.tool import Tool
-from fab.tools.category import Category
+from fab.tools import Tool
+from fab.category import Category
 from fab.tools.compiler import Compiler
 from fab.tools.linker import Linker
 from fab.tools.versioning import Fcm, Git, Subversion
 
 
 class ToolRepository(dict):
-    '''This class implements the tool repository. It stores a list of
-    tools for various categories. For each compiler, it will automatically
-    create a tool called "linker-{compiler-name}" which can be used for
-    linking with the specified compiler.
-    '''
-
-    _singleton: None | ToolRepository = None
-
-    def __new__(cls) -> ToolRepository:
-        '''Singleton access. Changes the value of _singleton so that the
-        constructor can verify that it is indeed called from here.
-        '''
-        if not cls._singleton:
-            cls._singleton = super().__new__(cls)
-
-        return cls._singleton
-
-    def __init__(self):
-        # Note that in this singleton pattern the constructor is called each
-        # time the instance is requested (since we overwrite __new__). But
-        # we only want to initialise the instance once, so let the constructor
-        # not do anything if the singleton already exists:
-        # pylint: disable=too-many-locals
-        if ToolRepository._singleton:
-            return
-
-        self._logger = logging.getLogger(__name__)
-        super().__init__()
+    """
+    Stores a list of tools for various categories.
+
+    For each compiler, it will automatically create a tool called
+    "linker-{compiler-name}" which can be used for linking with the specified
+    compiler.
+    """
+    def __new__(cls: Type[ToolRepository]) -> ToolRepository:
+        """
+        Returns a singleton object of this class.
+
+        Construction happens here and there is no constructor.
+        """
+        cls._singleton: ToolRepository
+        if hasattr(cls, '_singleton'):
+            return cls._singleton
+
+        singleton = super().__new__(cls)
+        cls._singleton = singleton
 
         # Create the list that stores all tools for each category:
         for category in Category:
-            self[category] = []
+            singleton[category] = []
 
         # Add the FAB default tools:
         # TODO: sort the defaults so that they actually work (since not all
         # tools FAB knows about are available). For now, disable Fpp:
         # We get circular dependencies if imported at top of the file:
         # pylint: disable=import-outside-toplevel
-        from fab.tools import (Ar, Cpp, CppFortran, Gcc, Gfortran,
-                               Icc, Ifort, Psyclone, Rsync)
+        from fab.tools.ar import Ar
+        from fab.tools.compiler import Gcc, Gfortran, Icc, Ifort
+        from fab.tools.preprocessor import Cpp, CppFortran
+        from fab.tools.psyclone import Psyclone
+        from fab.tools.rsync import Rsync
 
-        for cls in [Gcc, Icc, Gfortran, Ifort, Cpp, CppFortran,
-                    Fcm, Git, Subversion, Ar, Psyclone, Rsync]:
-            self.add_tool(cls())
+        for tool in [Gcc, Icc, Gfortran, Ifort, Cpp, CppFortran,
+                     Fcm, Git, Subversion, Ar, Psyclone, Rsync]:
+            singleton.add_tool(tool())
 
         from fab.tools.compiler_wrapper import Mpif90, Mpicc
-        all_fc = self[Category.FORTRAN_COMPILER][:]
+        all_fc = singleton[Category.FORTRAN_COMPILER][:]
         for fc in all_fc:
             mpif90 = Mpif90(fc)
-            self.add_tool(mpif90)
+            singleton.add_tool(mpif90)
 
-        all_cc = self[Category.C_COMPILER][:]
+        all_cc = singleton[Category.C_COMPILER][:]
         for cc in all_cc:
             mpicc = Mpicc(cc)
-            self.add_tool(mpicc)
+            singleton.add_tool(mpicc)
+
+        return singleton
 
     def add_tool(self, tool: Tool):
         '''Creates an instance of the specified class and adds it