Merge pull request #177 from Algebraic-Programming/develop

Release v0.7.0 into the main branch.

Highlights:

 1. This release re-implements the nonblocking ALP/GraphBLAS backend by Mastoras
    et al. (GrAPL/IPDPSW '22, TACO '23) on the latest ALP code base. The use of
    the nonblocking backend for some algorithms results in multiple-factor
    speedups versus standard blocking execution as well as versus external
    industry-standard frameworks. This includes Eigen, which, like nonblocking
    ALP/GraphBLAS, perform cross-operation fusion. Simply compile your ALP
    programs using `grbcxx -b nonblocking`, and enjoy the speedups!

 2. We also introduce a new programming interface to the ALP software stack that
    allows vertex-centric programming in addition to programming using
    generalised sparse linear algebra. This new interface, ALP/Pregel,
    translates vertex-centric programs to standard ALP/GraphBLAS primitives
    during compilation, and thus benefits of all automatic optimisations
    included with the ALP software stack.

 3. Support for software prefetching during `vxm` and `mxv` has been added to
    the `reference` and `reference_omp` backends. Since optimal prefetch
    settings and its overall effectiveness relies strongly on 1) the structure
    of the sparse matrices and graphs considered as well as on 2) the algorithms
    used on those data, this new feature is turned off by default. To use it,
    please enable it via `include/graphblas/reference/config.hpp` and tune the
    there-defined prefetch distances.

 4. Finally, this release includes another new backend, the `hyperdags` backend.
    A program compiled with this backend will, after execution, dump a HyperDAG
    representation of the ALP computations that the program executed.

More detailed changes since v0.6 are noted in the changelog.

This release has been thoroughly tested using various combinations of:
- OS: Fedora 31, CentOS Stream, Ubuntu 20.04 LTS;
- GCC: 8.5.0, 9.3.1, 9.4.0, and 12.2.0;
- MPI: MPICH 3.3.2, MPICH 3.4.2, and OpenMPI 4.1.1;
- Linux kernels: 4.18.0, 5.4.0, and 5.8.18;
- Architectures: x86_64 and ARMv8.2.

.github/workflows/dbg_smoke.yml

@@ -0,0 +1,42 @@
+
+name: debug-smoke-tests
+
+on: [push]
+
+env:
+  BUILD_TYPE: Debug
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Install required packages
+      run: sudo apt-get install -y libnuma-dev
+
+    - name: Configure
+      run: mkdir build && cd build && ../bootstrap.sh --prefix=../install --debug-build
+
+    - name: Build
+      working-directory: ${{github.workspace}}/build
+      run: make -j4
+
+    - name: Install
+      working-directory: ${{github.workspace}}/build
+      run: make -j4 install
+
+    - name: Test
+      working-directory: ${{github.workspace}}/build
+      run: make -j4 smoketests &> smoketests.log
+
+    - name: Check
+      working-directory: ${{github.workspace}}/build
+      run: ../tests/summarise.sh smoketests.log
+
+    - name: DumpLogOnFailure
+      if: failure()
+      working-directory: ${{github.workspace}}/build
+      run: cat smoketests.log
+

.github/workflows/smoke.yml

@@ -1,5 +1,5 @@
 
-name: smoke-tests
+name: release-smoke-tests
 
 on: [push]
 
@@ -35,3 +35,8 @@ jobs:
       working-directory: ${{github.workspace}}/build
       run: ../tests/summarise.sh smoketests.log
 
+    - name: DumpLogOnFailure
+      if: failure()
+      working-directory: ${{github.workspace}}/build
+      run: cat smoketests.log
+

.gitlab-ci.yml

@@ -30,7 +30,7 @@ stages:
 #    exclude:
 #      - build/**/*.o
 #      - build/**/*.o.d
-#    expire_in: 30 minutes
+#    expire_in: 80 minutes
 
 
 #build_debug_centos_8:
@@ -122,13 +122,16 @@ build_test:
     - apt update && apt -y install make cmake libnuma-dev coreutils
   script:
     - mkdir -p install build && cd ./build && ../bootstrap.sh --prefix=../install && make -j$(nproc) build_tests_all
+    - strip -s $(find tests/unit/ -type f -executable -print) $(find tests/smoke/ -type f -executable -print) $(find tests/performance/ -type f -executable -print)
   artifacts:
     paths:
       - build/
     exclude:
       - build/**/*.o
       - build/**/*.o.d
-    expire_in: 30 minutes
+      - build/**/CMakeFiles
+      - build/**/*.dir
+    expire_in: 80 minutes
 
 
 build_debug2_tests:
@@ -222,7 +225,7 @@ build_debug:
     exclude:
       - build/**/*.o
       - build/**/*.o.d
-    expire_in: 30 minutes
+    expire_in: 43 minutes
 
 
 test_smoke_debug:

CMakeLists.txt

@@ -27,7 +27,7 @@
 cmake_minimum_required( VERSION 3.13 )
 
 set( MAJORVERSION 0 )
-set( MINORVERSION 6 )
+set( MINORVERSION 7 )
 set( BUGVERSION 0 )
 set( VERSION "${MAJORVERSION}.${MINORVERSION}.${BUGVERSION}" )
 
@@ -51,6 +51,8 @@ endif()
 # to choose backends and dependencies
 option( WITH_REFERENCE_BACKEND "With Reference backend" ON )
 option( WITH_OMP_BACKEND "With OMP backend" ON )
+option( WITH_HYPERDAGS_BACKEND "With Hyperdags backend" ON )
+option( WITH_NONBLOCKING_BACKEND "With Nonblocking backend" ON )
 option( WITH_NUMA "With NUMA support" ON )
 option( LPF_INSTALL_PATH "Path to the LPF tools for the BSP1D and Hybrid backends" OFF )
 # the following options depend on LPF_INSTALL_PATH being set
@@ -61,6 +63,9 @@ LPF_INSTALL_PATH set)" ON LPF_INSTALL_PATH OFF
 cmake_dependent_option( WITH_HYBRID_BACKEND "Also build the Hybrid backend \
 (needs LPF_INSTALL_PATH set)" ON LPF_INSTALL_PATH OFF
 )
+# other dependent options
+cmake_dependent_option( WITH_HYPERDAGS_BACKEND "Building the Hyperdags backend needs \
+	WITH_HYPERDAGS_USING set" ON WITH_HYPERDAGS_USING OFF )
 # to customize build flags for either backends or tests
 option( COMMON_COMPILE_DEFINITIONS
 	"Compilation definitions for BOTH backends and tests; they override the defaults"
@@ -117,6 +122,7 @@ endif()
 
 if( NOT WITH_REFERENCE_BACKEND AND
 	NOT WITH_OMP_BACKEND AND
+	NOT WITH_NONBLOCKING_BACKEND AND
 	NOT WITH_BSP1D_BACKEND AND
 	NOT WITH_HYBRID_BACKEND )
 	message( FATAL_ERROR "At least one backend should be enabled")
@@ -188,13 +194,18 @@ endif()
 # by default no headers are built
 set( WITH_REFERENCE_BACKEND_HEADERS OFF )
 set( WITH_OMP_BACKEND_HEADERS OFF )
+set( WITH_HYPERDAGS_BACKEND_HEADERS OFF )
 
 # activate headers based on requested backends
-if( WITH_REFERENCE_BACKEND OR WITH_BSP1D_BACKEND )
-	# both reference and bsp1d backends need reference headers
+if( WITH_REFERENCE_BACKEND OR WITH_BSP1D_BACKEND OR WITH_NONBLOCKING_BACKEND )
+	# reference, bsp1d and nonblocking backends need reference headers
 	set( WITH_REFERENCE_BACKEND_HEADERS ON )
 endif()
 
+if( WITH_HYPERDAGS_BACKEND )
+	set( WITH_HYPERDAGS_BACKEND_HEADERS ON )
+endif()
+
 if( WITH_OMP_BACKEND OR WITH_HYBRID_BACKEND )
 	# both reference_omp and hynrid backends need reference headers
 	set( WITH_OMP_BACKEND_HEADERS ON )
@@ -218,13 +229,28 @@ add_subdirectory( examples )
 
 ### DOXYGEN DOCUMENTATION GENERATION
 
-set( DOCS_DIR "${PROJECT_SOURCE_DIR}/docs/code" )
+set( DOCS_DIR "${PROJECT_SOURCE_DIR}/docs/developer" )
 add_custom_command( OUTPUT "${DOCS_DIR}"
-	COMMAND bash -c "if [[ ! -d docs/code ]]; then doxygen docs/doxy.conf &> doxygen.log; fi"
+	COMMAND bash -c "doxygen docs/doxy.conf &> doxygen-developer.log;"
 	WORKING_DIRECTORY "${PROJECT_SOURCE_DIR}"
 	DEPENDS "${PROJECT_SOURCE_DIR}/docs/doxy.conf"
 	COMMENT "producing code documentation in ${DOCS_DIR}"
 	VERBATIM
 	#USES_TERMINAL
 )
-add_custom_target( docs DEPENDS "${DOCS_DIR}" )
+add_custom_target( devdocs DEPENDS "${DOCS_DIR}" )
+
+set( PUBLIC_DOCS_DIR "${PROJECT_SOURCE_DIR}/docs/user" )
+add_custom_command( OUTPUT "${PUBLIC_DOCS_DIR}"
+	COMMAND bash -c "doxygen docs/user.conf &> doxygen-user.log;"
+	WORKING_DIRECTORY "${PROJECT_SOURCE_DIR}"
+	DEPENDS "${PROJECT_SOURCE_DIR}/docs/user.conf"
+	COMMENT "producing public code documentation in ${PUBLIC_DOCS_DIR}"
+	VERBATIM
+)
+add_custom_target( userdocs DEPENDS "${PUBLIC_DOCS_DIR}" )
+add_custom_target( docs )
+add_dependencies( docs userdocs devdocs )
+
+message( "Compiling with the following backends: ${AVAILABLE_BACKENDS}\n" )
+

NOTICE

@@ -29,6 +29,8 @@ to Huawei Technologies Co., Ltd. or one of its subsidiaries:
 
  - Auke Booij, Huawei Technologies Switzerland AG; 2021.
 
+ - Anders Hansson, Huawei Technologies Switzerland AG; 2022-2023.
+
 The experimental banshee backend has been developed in collaboration with
 Prof. Luca Benini at ETH Zuerich and his group. In particular this backend
 is with great thanks due to Dan, Paul Scheffler, Fabian Schuiki, and Samuel