Fix GitLab CI + small improvements (#202)

This MR fixes the following issues encountered by the internal CI:
- the LPF tests could fail on gitlab runners that are not large enough;
- some CI coverage tests were skipped due to the missing CMake `DATASET_DIR`
- compilation of the SparseBLAS was always attempted no matter which backends were enabled
- fix sporadic bug of Coverage with OpenMP.

What is a "large enough" runner is undefined-- the documentation instead now mentions the configuration used for our internal CI, where this CI pipeline consistently passes, and thus describes what is sufficiently large.

The selection of a large-enough runner *may* require a specific (set of) runner tags that depend on your CI configuration. This MR allows for such tags to be made known via a site-specific (fixed global) variable, `LPF_PREFERRED_RUNNERS`, which may be set within your GitLab CI configuration. This way, new GitLab CI deployments should work out-of-the-box if the runners already happen to be "large enough", while if the explicit selection of "large-enough" runners is required, it is now easy to do so.

Thanks to @byjtew and @alberto-scolari for contributing this MR!

.gitlab-ci.yml

@@ -180,7 +180,6 @@ build_tests_buildtype_debug_sym_debug:
 build_tests_sym_debug:
   rules:
     - if: $EXTRA_TESTS_ENABLED == "yes"
-      when: on_success
   script:
     - mkdir -p install build && cd build && cmake -DCMAKE_INSTALL_PREFIX=../install -DCMAKE_CXX_FLAGS=-D_DEBUG
       -DCMAKE_C_FLAGS=-D_DEBUG -DLPF_INSTALL_PATH=${LPF_PATH} -DCMAKE_BUILD_TYPE=Release ../ && make -j$(nproc) build_tests_all
@@ -240,7 +239,6 @@ gitleaks:
 tests_performance:
   rules:
     - if: $EXTRA_TESTS_ENABLED == "yes"
-      when: on_success
   needs: [build_test]
   script:
     - cd ./build && make -j$(nproc) performancetests &> performancetests.log
@@ -249,7 +247,6 @@ tests_performance:
 tests_unit_buildtype_debug:
   rules:
     - if: $EXTRA_TESTS_ENABLED == "yes"
-      when: on_success
   needs: [build_test_buildtype_debug]
   script:
     - cd ./build && make -j$(nproc) unittests &> unittests.log
@@ -276,19 +273,33 @@ build_test_lpf:
       - build/**/*.dir
     expire_in: 2 hours
 
-tests_unit_lpf:
-  rules:
-    - if: $LPF_TESTS_ENABLED == "yes"
-      when: on_success
+# common sections for  LPF unit tests
+.tests_unit_lpf:
   needs: [build_test_lpf]
   script:
     - cd ./build && make -j$(nproc) tests_unit &> unittests.log
     - ../tests/summarise.sh unittests.log
 
+# this job triggers in internal CI, where LPF tests run better on runners
+# with a given tag $LPF_PREFERRED_RUNNERS_TAG
+tests_unit_lpf_preferred:
+  rules:
+    - if: $LPF_TESTS_ENABLED == "yes" && $LPF_PREFERRED_RUNNERS == "yes"
+  tags:
+    - docker
+    - $LPF_PREFERRED_RUNNERS_TAG
+  extends: .tests_unit_lpf
+
+# if runners with a specific tag are not present, run this job
+# attention: it may timeout
+tests_unit_lpf_generic:
+  rules:
+    - if: $LPF_TESTS_ENABLED == "yes" && $LPF_PREFERRED_RUNNERS != "yes"
+  extends: .tests_unit_lpf
+
 tests_smoke_lpf:
   rules:
     - if: $LPF_TESTS_ENABLED == "yes"
-      when: on_success
   needs: [build_test_lpf]
   script:
     - cd ./build && make -j$(nproc) tests_smoke &> smoketests.log
@@ -297,7 +308,6 @@ tests_smoke_lpf:
 test_installation_lpf:
   rules:
     - if: $LPF_TESTS_ENABLED == "yes"
-      when: on_success
   needs: [build_test_lpf]
   script:
     - cd ./build && make -j$(nproc) install
@@ -307,7 +317,6 @@ test_installation_lpf:
 build_test_gcc_versions:
   rules:
     - if: $MULTIPLE_COMPILERS_ENABLED == "yes"
-      when: on_success
   image: ${CI_REGISTRY}/${CI_PROJECT_PATH}/lpf-ubuntu-22.04-gcc-clang
   parallel:
       matrix:
@@ -345,11 +354,14 @@ coverage_matrix:
     - echo "-- NONBLOCKING=${backends_array[3]}"
 
     - rm -rf build install && mkdir -p install build && cd build
-    - cmake -DCMAKE_INSTALL_PREFIX='../install' -DCMAKE_BUILD_TYPE=Coverage
+    - cmake -DCMAKE_INSTALL_PREFIX='../install' 
+      -DCMAKE_BUILD_TYPE=Coverage
+      -DDATASETS_DIR=${ALP_DATASETS}
       -DWITH_HYPERDAGS_BACKEND=${backends_array[0]}
       -DWITH_REFERENCE_BACKEND=${backends_array[1]}
       -DWITH_OMP_BACKEND=${backends_array[2]}
       -DWITH_NONBLOCKING_BACKEND=${backends_array[3]} ..
+    - make -j$(nproc)
     - make -j$(nproc) unittests
 # for each job (i.e., each backend), generate a separate JSON to me merged later
 # (gcovr merges only JSON files)
@@ -361,6 +373,7 @@ coverage_matrix:
   artifacts:
     paths:
       - COVERAGE_${CI_JOB_ID}.json
+    expire_in: 4 weeks
 
 cobertura_coverage_report:
   rules:

cmake/CompileFlags.cmake

@@ -71,7 +71,7 @@ set_valid_string( COMMON_OPTS_Debug "${COMMON_COMPILE_OPTIONS}"
 	"${COMMON_OPTS};-fno-omit-frame-pointer"
 )
 set_valid_string( COMMON_OPTS_Coverage "${COMMON_COMPILE_OPTIONS}"
-	"${COMMON_OPTS};-fprofile-arcs;-ftest-coverage"
+	"${COMMON_OPTS};-fprofile-arcs;-ftest-coverage;-fprofile-update=atomic"
 )
 
 add_library( common_flags INTERFACE )

docs/Build_and_test_infra.md

@@ -835,6 +835,7 @@ During configuration this file generates a report with all compilation flags for
 the various target types and categories/modes.
 
 # Reproducible Builds
+
 To ease building and deploying ALP/GraphBLAS, dedicated Docker images can be
 built from the `Dockerfile`s in the *ALP/ReproducibleBuild* repository
 
@@ -853,6 +854,9 @@ Indeed, the file [`.gitlab-ci.yml`](../.gitlab-ci.yml) describes the CI jobs
 that internally test ALP/GraphBLAS via [GitLab](https://about.gitlab.com/),
 which is available [open source](https://about.gitlab.com/install/).
 
+The here described image and CI pipeline are confirmed to work with x86 runners
+with 24 virtual CPU cores and 32GB RAM.
+
 # The coverage infrastructure
 
 The *Coverage* build type stores coverage information into machine- or human-readable

src/transition/CMakeLists.txt

@@ -20,43 +20,46 @@
 
 assert_defined_variables( WITH_REFERENCE_BACKEND WITH_OMP_BACKEND )
 
-add_library( sparseblas_static STATIC
-	${CMAKE_CURRENT_SOURCE_DIR}/sparseblas.cpp
-)
+if( WITH_REFERENCE_BACKEND )
+	add_library( sparseblas_static STATIC
+		${CMAKE_CURRENT_SOURCE_DIR}/sparseblas.cpp
+	)
 
-set_target_properties( sparseblas_static PROPERTIES
-	OUTPUT_NAME "sparseblas"
-	ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/shmem"
-)
+	set_target_properties( sparseblas_static PROPERTIES
+		OUTPUT_NAME "sparseblas"
+		ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/shmem"
+	)
 
-target_link_libraries( sparseblas_static PUBLIC backend_reference transition )
+	target_link_libraries( sparseblas_static PUBLIC backend_reference transition )
 
-target_link_libraries( sparseblas_static PRIVATE backend_flags )
+	target_link_libraries( sparseblas_static PRIVATE backend_flags )
 
-add_dependencies( libs sparseblas_static )
+	add_dependencies( libs sparseblas_static )
 
-install( TARGETS sparseblas_static
-	EXPORT GraphBLASTargets
-	ARCHIVE DESTINATION "${SHMEM_BACKEND_INSTALL_DIR}"
-)
+	install( TARGETS sparseblas_static
+		EXPORT GraphBLASTargets
+		ARCHIVE DESTINATION "${SHMEM_BACKEND_INSTALL_DIR}"
+	)
+endif( WITH_REFERENCE_BACKEND )
 
-add_library( sparseblas_omp_static STATIC
-	${CMAKE_CURRENT_SOURCE_DIR}/sparseblas.cpp
-)
+if( WITH_OMP_BACKEND )
+	add_library( sparseblas_omp_static STATIC
+		${CMAKE_CURRENT_SOURCE_DIR}/sparseblas.cpp
+	)
 
-set_target_properties( sparseblas_omp_static PROPERTIES
-	OUTPUT_NAME "sparseblas_omp"
-	ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/shmem"
-)
+	set_target_properties( sparseblas_omp_static PROPERTIES
+		OUTPUT_NAME "sparseblas_omp"
+		ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/shmem"
+	)
 
-target_link_libraries( sparseblas_omp_static PUBLIC backend_reference_omp transition )
+	target_link_libraries( sparseblas_omp_static PUBLIC backend_reference_omp transition )
 
-target_link_libraries( sparseblas_omp_static PRIVATE backend_flags )
+	target_link_libraries( sparseblas_omp_static PRIVATE backend_flags )
 
-add_dependencies( libs sparseblas_omp_static )
-
-install( TARGETS sparseblas_omp_static
-	EXPORT GraphBLASTargets
-	ARCHIVE DESTINATION "${SHMEM_BACKEND_INSTALL_DIR}"
-)
+	add_dependencies( libs sparseblas_omp_static )
 
+	install( TARGETS sparseblas_omp_static
+		EXPORT GraphBLASTargets
+		ARCHIVE DESTINATION "${SHMEM_BACKEND_INSTALL_DIR}"
+	)
+endif()