Redesign tutorials CI/CD framework

Use standard CMake targets to configure tutorials. Mark figure files
as dependencies of the Jupyter notebooks that actually use them.
Make python and html targets depend on the original tutorial file in
the source directory. Convert solution cells to hidden solutions.

doc/tutorials/CMakeLists.txt

@@ -19,47 +19,50 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 
-# Register a new tutorial target that depends on a list of files that need to be
-# copied from the source directory to the build directory. Make the tutorial
-# target a dependency of target `tutorials`.
-function(configure_tutorial_target)
-  cmake_parse_arguments(CONFIGURE "" "TARGET" "DEPENDS" ${ARGN})
-  set(TARGET_DEPENDENCIES "")
-  foreach(filepath ${CONFIGURE_DEPENDS})
-    list(APPEND TARGET_DEPENDENCIES "${CMAKE_CURRENT_BINARY_DIR}/${filepath}")
-    add_custom_command(
-      OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${filepath}"
-      DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/${filepath}"
-      COMMAND
-        ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/${filepath}"
-        "${CMAKE_CURRENT_BINARY_DIR}/${filepath}")
-  endforeach(filepath)
-  add_custom_target(${CONFIGURE_TARGET} DEPENDS ${TARGET_DEPENDENCIES})
-  add_dependencies(tutorials ${CONFIGURE_TARGET})
-endfunction(configure_tutorial_target)
+add_custom_target(tutorials)
+add_custom_target(tutorials_html)
+add_custom_target(tutorials_python)
+
+configure_file(Readme.md ${CMAKE_CURRENT_BINARY_DIR} COPYONLY)
+configure_file(convert.py ${CMAKE_CURRENT_BINARY_DIR})
 
 # Convert a tutorial to the Python and HTML formats. Make these files
 # dependencies of targets `tutorials_html` and `tutorials_python`.
-function(NB_EXPORT)
+function(ESPRESSO_ADD_TUTORIAL)
   cmake_parse_arguments(NB_EXPORT "HTML_RUN" "FILE;TARGET;SUFFIX"
-                        "VAR_SUBST;ADD_SCRIPTS" ${ARGN})
+                        "VAR_SUBST;ADD_SCRIPTS;DEPENDS" ${ARGN})
+
+  # create target if it doesn't already exist
+  set(TUTORIAL_TARGET ${NB_EXPORT_TARGET})
+  if(NOT TARGET ${TUTORIAL_TARGET})
+    add_custom_target(${TUTORIAL_TARGET})
+  endif()
 
-  set(NB_FILE "${CMAKE_CURRENT_BINARY_DIR}/${NB_EXPORT_FILE}")
+  # deploy dependencies
+  set(TARGET_DEPENDENCIES "")
+  foreach(RELPATH ${NB_EXPORT_DEPENDS})
+    configure_file(${RELPATH} ${RELPATH} COPYONLY)
+    list(APPEND TARGET_DEPENDENCIES ${CMAKE_CURRENT_BINARY_DIR}/${RELPATH})
+  endforeach()
+
+  set(NB_FILE ${CMAKE_CURRENT_SOURCE_DIR}/${NB_EXPORT_FILE})
+  set(IPYNB_FILE ${CMAKE_CURRENT_BINARY_DIR}/${NB_EXPORT_FILE})
   set(DEPENDENCY_OF_TARGET "${NB_EXPORT_TARGET}")
-  if(NOT "${NB_EXPORT_SUFFIX}" STREQUAL "")
+  if(DEFINED NB_EXPORT_SUFFIX AND NOT "${NB_EXPORT_SUFFIX}" STREQUAL "")
     set(NB_EXPORT_TARGET "${NB_EXPORT_TARGET}_${NB_EXPORT_SUFFIX}")
   endif()
   cmake_path(GET NB_FILE STEM NB_FILE_STEM)
   cmake_path(GET NB_FILE EXTENSION NB_FILE_EXT)
-  set(HTML_FILE "${NB_FILE_STEM}.html")
-  set(PY_FILE "${NB_FILE_STEM}.py")
+  set(HTML_FILE "${CMAKE_CURRENT_BINARY_DIR}/${NB_FILE_STEM}.html")
+  set(PY_FILE "${CMAKE_CURRENT_BINARY_DIR}/${NB_FILE_STEM}.py")
 
   if(NB_EXPORT_HTML_RUN)
     set(NB_FILE_RUN "${NB_FILE_STEM}.run${NB_FILE_EXT}")
     add_custom_command(
       OUTPUT ${NB_FILE_RUN}
-      DEPENDS
-        "${NB_FILE};${NB_EXPORT_ADD_SCRIPTS};${CMAKE_BINARY_DIR}/doc/tutorials/convert.py;${CMAKE_BINARY_DIR}/testsuite/scripts/importlib_wrapper.py"
+      DEPENDS ${NB_FILE} ${NB_EXPORT_ADD_SCRIPTS} ${CMAKE_BINARY_DIR}/pypresso
+              ${CMAKE_BINARY_DIR}/doc/tutorials/convert.py
+              ${CMAKE_BINARY_DIR}/testsuite/scripts/importlib_wrapper.py
       COMMAND
         ${CMAKE_BINARY_DIR}/pypresso
         ${CMAKE_BINARY_DIR}/doc/tutorials/convert.py ci --execute
@@ -71,7 +74,16 @@ function(NB_EXPORT)
   endif()
 
   add_custom_command(
-    OUTPUT ${HTML_FILE} DEPENDS ${NB_FILE_RUN};${NB_EXPORT_ADD_SCRIPTS}
+    OUTPUT ${IPYNB_FILE}
+    DEPENDS ${NB_FILE} ${CMAKE_BINARY_DIR}/pypresso
+            ${CMAKE_BINARY_DIR}/doc/tutorials/convert.py
+    COMMAND ${CMAKE_COMMAND} -E copy ${NB_FILE} ${IPYNB_FILE}
+    COMMAND
+      ${CMAKE_BINARY_DIR}/pypresso ${CMAKE_BINARY_DIR}/doc/tutorials/convert.py
+      cells --to-md ${IPYNB_FILE})
+
+  add_custom_command(
+    OUTPUT ${HTML_FILE} DEPENDS ${NB_FILE_RUN} ${NB_EXPORT_ADD_SCRIPTS}
     COMMAND ${IPYTHON_EXECUTABLE} nbconvert --to "html" --output ${HTML_FILE}
             ${NB_FILE_RUN})
 
@@ -80,17 +92,17 @@ function(NB_EXPORT)
     COMMAND ${IPYTHON_EXECUTABLE} nbconvert --to "python" --output ${PY_FILE}
             ${NB_FILE})
 
+  add_custom_target("${NB_EXPORT_TARGET}_deps" DEPENDS ${TARGET_DEPENDENCIES}
+                                                       ${IPYNB_FILE})
   add_custom_target("${NB_EXPORT_TARGET}_html" DEPENDS ${HTML_FILE}
-                                                       ${DEPENDENCY_OF_TARGET})
-  add_custom_target("${NB_EXPORT_TARGET}_python"
-                    DEPENDS ${PY_FILE} ${DEPENDENCY_OF_TARGET})
+                                                       ${TUTORIAL_TARGET})
+  add_custom_target("${NB_EXPORT_TARGET}_python" DEPENDS ${PY_FILE}
+                                                         ${TUTORIAL_TARGET})
+  add_dependencies(${TUTORIAL_TARGET} "${NB_EXPORT_TARGET}_deps")
+  add_dependencies(tutorials ${TUTORIAL_TARGET})
   add_dependencies(tutorials_html "${NB_EXPORT_TARGET}_html")
   add_dependencies(tutorials_python "${NB_EXPORT_TARGET}_python")
-endfunction(NB_EXPORT)
-
-add_custom_target(tutorials)
-add_custom_target(tutorials_html)
-add_custom_target(tutorials_python)
+endfunction()
 
 # Here: add new directory
 add_subdirectory(lennard_jones)
@@ -109,6 +121,3 @@ add_subdirectory(widom_insertion)
 add_subdirectory(electrodes)
 add_subdirectory(grand_canonical_monte_carlo)
 add_subdirectory(mlip)
-
-configure_file(Readme.md ${CMAKE_CURRENT_BINARY_DIR} COPYONLY)
-configure_file(convert.py ${CMAKE_CURRENT_BINARY_DIR})

doc/tutorials/active_matter/CMakeLists.txt

@@ -1,5 +1,5 @@
 #
-# Copyright (C) 2017-2022 The ESPResSo project
+# Copyright (C) 2017-2025 The ESPResSo project
 #
 # This file is part of ESPResSo.
 #
@@ -17,8 +17,6 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 
-configure_tutorial_target(
-  TARGET tutorial_am DEPENDS active_matter.ipynb figures/friction.svg
-  figures/pusher-puller.svg figures/geometry.svg)
-
-nb_export(TARGET tutorial_am FILE "active_matter.ipynb" HTML_RUN)
+espresso_add_tutorial(
+  TARGET tutorial_am FILE "active_matter.ipynb" HTML_RUN DEPENDS
+  figures/friction.svg figures/pusher-puller.svg figures/geometry.svg)

doc/tutorials/charged_system/CMakeLists.txt

@@ -1,5 +1,5 @@
 #
-# Copyright (C) 2016-2022 The ESPResSo project
+# Copyright (C) 2016-2025 The ESPResSo project
 #
 # This file is part of ESPResSo.
 #
@@ -17,6 +17,4 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 
-configure_tutorial_target(TARGET tutorial_crg DEPENDS charged_system.ipynb)
-
-nb_export(TARGET tutorial_crg FILE "charged_system.ipynb" HTML_RUN)
+espresso_add_tutorial(TARGET tutorial_crg FILE "charged_system.ipynb" HTML_RUN)

doc/tutorials/constant_pH/CMakeLists.txt

@@ -1,5 +1,5 @@
 #
-# Copyright (C) 2019-2022 The ESPResSo project
+# Copyright (C) 2019-2025 The ESPResSo project
 #
 # This file is part of ESPResSo.
 #
@@ -17,6 +17,4 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 
-configure_tutorial_target(TARGET tutorial_cph DEPENDS constant_pH.ipynb)
-
-nb_export(TARGET tutorial_cph SUFFIX "" FILE "constant_pH.ipynb" HTML_RUN)
+espresso_add_tutorial(TARGET tutorial_cph FILE "constant_pH.ipynb" HTML_RUN)

doc/tutorials/convert.py

@@ -32,7 +32,6 @@
 import sys
 import uuid
 sys.path.append('@CMAKE_SOURCE_DIR@/testsuite/scripts')
-import importlib_wrapper as iw
 
 
 SOLUTION_CELL_TOKEN = "# SOLUTION CELL"
@@ -186,6 +185,7 @@ def execute_notebook(nb, src, cell_separator, notebook_filepath):
     the notebook in a CI environment.
     """
     import nbconvert.preprocessors
+    import importlib_wrapper as iw
     notebook_dirname = os.path.dirname(notebook_filepath)
     # disable OpenGL GUI
     src_no_gui = iw.mock_es_visualization(src)
@@ -224,6 +224,7 @@ def handle_ci_case(args):
     disable_plot_interactivity(nb)
 
     if args.substitutions or args.execute:
+        import importlib_wrapper as iw
         # substitute global variables
         cell_separator = f'\n##{uuid.uuid4().hex}\n'
         src = cell_separator.join(get_code_cells(nb))

doc/tutorials/electrodes/CMakeLists.txt

@@ -1,5 +1,5 @@
 #
-# Copyright (C) 2020-2022 The ESPResSo project
+# Copyright (C) 2020-2025 The ESPResSo project
 #
 # This file is part of ESPResSo.
 #
@@ -17,10 +17,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 
-configure_tutorial_target(TARGET tutorial_electrodes DEPENDS
-                          electrodes_part1.ipynb electrodes_part2.ipynb)
-
-nb_export(TARGET tutorial_electrodes SUFFIX "1" FILE "electrodes_part1.ipynb"
-          HTML_RUN)
-nb_export(TARGET tutorial_electrodes SUFFIX "2" FILE "electrodes_part2.ipynb"
-          HTML_RUN)
+espresso_add_tutorial(TARGET tutorial_electrodes SUFFIX "1" FILE
+                      "electrodes_part1.ipynb" HTML_RUN)
+espresso_add_tutorial(TARGET tutorial_electrodes SUFFIX "2" FILE
+                      "electrodes_part2.ipynb" HTML_RUN)

doc/tutorials/electrokinetics/CMakeLists.txt

@@ -1,5 +1,5 @@
 #
-# Copyright (C) 2016-2022 The ESPResSo project
+# Copyright (C) 2016-2025 The ESPResSo project
 #
 # This file is part of ESPResSo.
 #
@@ -17,7 +17,5 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 
-configure_tutorial_target(TARGET tutorial_ek DEPENDS electrokinetics.ipynb
-                          figures/schlitzpore_3d.png)
-
-nb_export(TARGET tutorial_ek SUFFIX "" FILE "electrokinetics.ipynb" HTML_RUN)
+espresso_add_tutorial(TARGET tutorial_ek FILE "electrokinetics.ipynb" HTML_RUN
+                      DEPENDS figures/schlitzpore_3d.png)

doc/tutorials/error_analysis/CMakeLists.txt

@@ -1,5 +1,5 @@
 #
-# Copyright (C) 2021-2022 The ESPResSo project
+# Copyright (C) 2021-2025 The ESPResSo project
 #
 # This file is part of ESPResSo.
 #
@@ -17,10 +17,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 
-configure_tutorial_target(TARGET tutorial_err DEPENDS
-                          error_analysis_part1.ipynb error_analysis_part2.ipynb)
-
-nb_export(TARGET tutorial_err SUFFIX "1" FILE "error_analysis_part1.ipynb"
-          HTML_RUN)
-nb_export(TARGET tutorial_err SUFFIX "2" FILE "error_analysis_part2.ipynb"
-          HTML_RUN)
+espresso_add_tutorial(TARGET tutorial_err SUFFIX "1" FILE
+                      "error_analysis_part1.ipynb" HTML_RUN)
+espresso_add_tutorial(TARGET tutorial_err SUFFIX "2" FILE
+                      "error_analysis_part2.ipynb" HTML_RUN)

doc/tutorials/ferrofluid/CMakeLists.txt

@@ -1,5 +1,5 @@
 #
-# Copyright (C) 2018-2022 The ESPResSo project
+# Copyright (C) 2018-2025 The ESPResSo project
 #
 # This file is part of ESPResSo.
 #
@@ -17,12 +17,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 
-configure_tutorial_target(
-  TARGET tutorial_fe DEPENDS ferrofluid_part1.ipynb ferrofluid_part2.ipynb
-  ferrofluid_part3.ipynb figures/Electro-Steric_Stabilization.jpg
-  figures/Ferrofluid_Magnet_under_glass_edit.jpg figures/headtotailconf.png)
-
-nb_export(
+espresso_add_tutorial(
   TARGET
   tutorial_fe
   SUFFIX
@@ -32,11 +27,14 @@ nb_export(
   HTML_RUN
   VAR_SUBST
   "EQUIL_STEPS=100;EQUIL_ROUNDS=10;\"CI_DP3M_PARAMS={'cao':3,'r_cut':8.34,'mesh':[8,8,8],'alpha':0.2115,'tune':False}\""
-)
-nb_export(TARGET tutorial_fe SUFFIX "2" FILE "ferrofluid_part2.ipynb" HTML_RUN
-          VAR_SUBST
-          "equil_steps=100;equil_rounds=10;alphas=[0,1,2,3,4.5,8];loops=100")
-nb_export(
+  DEPENDS
+  figures/Electro-Steric_Stabilization.jpg
+  figures/Ferrofluid_Magnet_under_glass_edit.jpg
+  figures/headtotailconf.png)
+espresso_add_tutorial(
+  TARGET tutorial_fe SUFFIX "2" FILE "ferrofluid_part2.ipynb" HTML_RUN
+  VAR_SUBST "equil_steps=100;equil_rounds=10;alphas=[0,1,2,3,4.5,8];loops=100")
+espresso_add_tutorial(
   TARGET tutorial_fe SUFFIX "3" FILE "ferrofluid_part3.ipynb" HTML_RUN
   VAR_SUBST
   "equil_steps=100;equil_rounds=10;alphas=[0,0.5,1,2,4,8];loops=200;loops_m=100"

doc/tutorials/grand_canonical_monte_carlo/CMakeLists.txt

@@ -1,5 +1,5 @@
 #
-# Copyright (C) 2023 The ESPResSo project
+# Copyright (C) 2023-2025 The ESPResSo project
 #
 # This file is part of ESPResSo.
 #
@@ -17,10 +17,8 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 
-configure_tutorial_target(
-  TARGET tutorial_grand_canonical_monte_carlo DEPENDS
-  grand_canonical_monte_carlo.ipynb figures/schematic.svg)
-
-nb_export(TARGET tutorial_grand_canonical_monte_carlo SUFFIX "" FILE
-          "grand_canonical_monte_carlo.ipynb" HTML_RUN VAR_SUBST
-          "\"p3m_params={'mesh':10,'cao':6,'r_cut':8.22}\"")
+espresso_add_tutorial(
+  TARGET tutorial_grand_canonical_monte_carlo FILE
+  "grand_canonical_monte_carlo.ipynb" HTML_RUN VAR_SUBST
+  "\"p3m_params={'mesh':10,'cao':6,'r_cut':8.22}\"" DEPENDS
+  figures/schematic.svg)

doc/tutorials/langevin_dynamics/CMakeLists.txt

@@ -1,5 +1,5 @@
 #
-# Copyright (C) 2021-2022 The ESPResSo project
+# Copyright (C) 2021-2025 The ESPResSo project
 #
 # This file is part of ESPResSo.
 #
@@ -17,6 +17,5 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 
-configure_tutorial_target(TARGET tutorial_ld DEPENDS langevin_dynamics.ipynb)
-
-nb_export(TARGET tutorial_ld SUFFIX "" FILE "langevin_dynamics.ipynb" HTML_RUN)
+espresso_add_tutorial(TARGET tutorial_ld FILE "langevin_dynamics.ipynb"
+                      HTML_RUN)

doc/tutorials/lattice_boltzmann/CMakeLists.txt

@@ -1,5 +1,5 @@
 #
-# Copyright (C) 2016-2022 The ESPResSo project
+# Copyright (C) 2016-2025 The ESPResSo project
 #
 # This file is part of ESPResSo.
 #
@@ -17,15 +17,11 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 
-configure_tutorial_target(
-  TARGET tutorial_lb DEPENDS lattice_boltzmann_theory.ipynb
-  lattice_boltzmann_poiseuille_flow.ipynb lattice_boltzmann_sedimentation.ipynb
-  figures/latticeboltzmann-grid.png
+espresso_add_tutorial(
+  TARGET tutorial_lb SUFFIX "1" FILE "lattice_boltzmann_theory.ipynb" HTML_RUN
+  DEPENDS figures/latticeboltzmann-grid.png
   figures/latticeboltzmann-momentumexchange.png)
-
-nb_export(TARGET tutorial_lb SUFFIX "1" FILE "lattice_boltzmann_theory.ipynb"
-          HTML_RUN)
-nb_export(TARGET tutorial_lb SUFFIX "2" FILE
-          "lattice_boltzmann_poiseuille_flow.ipynb" HTML_RUN)
-nb_export(TARGET tutorial_lb SUFFIX "3" FILE
-          "lattice_boltzmann_sedimentation.ipynb" HTML_RUN)
+espresso_add_tutorial(TARGET tutorial_lb SUFFIX "2" FILE
+                      "lattice_boltzmann_poiseuille_flow.ipynb" HTML_RUN)
+espresso_add_tutorial(TARGET tutorial_lb SUFFIX "3" FILE
+                      "lattice_boltzmann_sedimentation.ipynb" HTML_RUN)

doc/tutorials/lennard_jones/CMakeLists.txt

@@ -1,5 +1,5 @@
 #
-# Copyright (C) 2016-2022 The ESPResSo project
+# Copyright (C) 2016-2025 The ESPResSo project
 #
 # This file is part of ESPResSo.
 #
@@ -17,7 +17,4 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 
-configure_tutorial_target(TARGET tutorial_lj DEPENDS lennard_jones.ipynb)
-
-nb_export(TARGET tutorial_lj SUFFIX "" FILE "lennard_jones.ipynb" HTML_RUN
-          VAR_SUBST)
+espresso_add_tutorial(TARGET tutorial_lj FILE "lennard_jones.ipynb" HTML_RUN)