From 2c59cbc50811c5356a498defc58eba21fc5a8d93 Mon Sep 17 00:00:00 2001 From: Hunter Hansen <50837800+hhansen-bdai@users.noreply.github.com> Date: Wed, 17 Apr 2024 09:02:20 -0400 Subject: [PATCH] Adds X11 forwarding and yaml storage for Docker (#490) # Description This PR adds the capability to opt-into and use X11 Forwarding from within the container. All of the configuration for the compose network is in `x11.yaml`, the preparation happens in `./container.sh`. Deeper explanation: Instead of attempting to mount an .Xauth directly, we create one ourselves. The `Xauth` cookie is extracted and merged into a new .tmp file which is created on the local user's system. This file is then mounted inside the container,as is the X11 unix socket, and the DISPLAY envvar (as well as a few others to avoid problems) are passed from the host machine into the container. This should work reliably on a container running locally, and it has worked for me over ssh. However, given that ssh can make this quite a bit more complicated, this PR only seeks to support local x11 forwarding. **UPDATE**: I've also added some state persistence through `yq` and the file `.container.yaml`. I am using this to store settings we only want to ask once (`__ORBIT_X11_FORWARDING_ENABLED`) and state that needs to be used by multiple functions (`__ORBIT_TMP_XAUTH`). I added a few helper functions and and installation function to install `yq` for yaml operations. ## Type of change - New feature (non-breaking change which adds functionality) ## Checklist - [x] I have run the [`pre-commit` checks](https://pre-commit.com/) with `./orbit.sh --format` - [ ] I have made corresponding changes to the documentation - [x] My changes generate no new warnings - [ ] I have added tests that prove my fix is effective or that my feature works - [x] I have run all the tests with `./orbit.sh --test` and they pass - [ ] I have updated the changelog and the corresponding version in the extension's `config/extension.toml` file - [x] I have added my name to the `CONTRIBUTORS.md` or my name already exists there --------- Signed-off-by: Hunter Hansen <50837800+hhansen-bdai@users.noreply.github.com> Co-authored-by: Pascal Roth <57946385+pascal-roth@users.noreply.github.com> --- .dockerignore | 3 + .gitignore | 1 + docker/container.sh | 114 +++++++++++++++++++++++++- docker/docker-compose.yaml | 159 +++++++++++++++++-------------------- docker/x11.yaml | 36 +++++++++ 5 files changed, 224 insertions(+), 89 deletions(-) create mode 100644 docker/x11.yaml diff --git a/.dockerignore b/.dockerignore index 397c2c65dc..81bf4b49f1 100644 --- a/.dockerignore +++ b/.dockerignore @@ -13,6 +13,9 @@ docs/ *.tmp # ignore docker docker/exports/ +docker/.container.yaml +# ignore recordings +recordings/ # ignore __pycache__ **/__pycache__/ **/*.egg-info/ diff --git a/.gitignore b/.gitignore index 2292fd9c2f..43dd2d707b 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,7 @@ # Docker/Singularity **/*.sif docker/exports/ +docker/.container.yaml # IDE **/.idea/ diff --git a/docker/container.sh b/docker/container.sh index 3fb193d86d..5e92ab3877 100755 --- a/docker/container.sh +++ b/docker/container.sh @@ -13,6 +13,12 @@ tabs 4 # get script directory SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +STATEFILE="${SCRIPT_DIR}/.container.yaml" + +if ! [ -f "$STATEFILE" ]; then + touch $STATEFILE +fi + #== # Functions #== @@ -47,6 +53,34 @@ install_apptainer() { fi } +install_yq() { + # Installing yq to handle file parsing + # Installation procedure from here: https://github.com/mikefarah/yq + read -p "[INFO] Required 'yq' package could not be found. Would you like to install it via wget? (y/N)" yq_answer + if [ "$yq_answer" != "${yq_answer#[Yy]}" ]; then + sudo snap install yq + else + echo "[INFO] Exiting because yq was not installed" + exit + fi +} + +set_statefile_variable() { + # Stores key $1 with value $2 in yaml $STATEFILE + yq -i '.["'"$1"'"] = "'"$2"'"' $STATEFILE +} + +load_statefile_variable() { + # Loads key $1 from yaml $STATEFILE as an envvar + # If key does not exist, the loaded var will equal "null" + eval $1="$(yq ".$1" $STATEFILE)" +} + +delete_statefile_variable() { + # Deletes key $1 from yaml $STATEFILE + yq -i "del(.$1)" $STATEFILE +} + # Function to check docker versions # If docker version is more than 25, the script errors out. check_docker_version() { @@ -85,6 +119,8 @@ resolve_image_extension() { container_profile="base" fi + add_yamls="--file docker-compose.yaml" + add_profiles="--profile $container_profile" # We will need .env.base regardless of profile add_envs="--env-file .env.base" @@ -125,6 +161,75 @@ check_singularity_image_exists() { fi } +install_xauth() { + # check if xauth is installed + read -p "[INFO] xauth is not installed. Would you like to install it via apt? (y/N) " xauth_answer + if [ "$xauth_answer" != "${xauth_answer#[Yy]}" ]; then + sudo apt update && sudo apt install xauth + else + echo "[INFO] Did not install xauth. Full X11 forwarding not enabled." + fi +} + +# This is modeled after Rocker's x11 forwarding extension +# https://github.com/osrf/rocker +configure_x11() { + if ! command -v xauth &> /dev/null; then + install_xauth + fi + load_statefile_variable __ORBIT_TMP_XAUTH + # Create temp .xauth file to be mounted in the container + if [ "$__ORBIT_TMP_XAUTH" = "null" ]; then + __ORBIT_TMP_XAUTH=$(mktemp --suffix=".xauth") + set_statefile_variable __ORBIT_TMP_XAUTH $__ORBIT_TMP_XAUTH + # Extract MIT-MAGIC-COOKIE for current display | Change the 'connection family' to FamilyWild (ffff) | merge into tmp .xauth file + # https://www.x.org/archive/X11R6.8.1/doc/Xsecurity.7.html#toc3 + xauth_cookie= xauth nlist ${DISPLAY} | sed -e s/^..../ffff/ | xauth -f $__ORBIT_TMP_XAUTH nmerge - + fi + # Export here so it's an envvar for the called Docker commands + export __ORBIT_TMP_XAUTH + add_yamls="$add_yamls --file x11.yaml " + # TODO: Add check to make sure Xauth file is correct +} + +x11_check() { + load_statefile_variable __ORBIT_X11_FORWARDING_ENABLED + if [ "$__ORBIT_X11_FORWARDING_ENABLED" = "null" ]; then + echo "[INFO] X11 forwarding from the Orbit container is off by default." + echo "[INFO] It will fail if there is no display, or this script is being run via ssh without proper configuration." += read -p "Would you like to enable it? (y/N) " x11_answer + if [ "$x11_answer" != "${x11_answer#[Yy]}" ]; then + __ORBIT_X11_FORWARDING_ENABLED=1 + set_statefile_variable __ORBIT_X11_FORWARDING_ENABLED 1 + echo "[INFO] X11 forwarding is enabled from the container." + else + __ORBIT_X11_FORWARDING_ENABLED=0 + set_statefile_variable __ORBIT_X11_FORWARDING_ENABLED 0 + echo "[INFO] X11 forwarding is disabled from the container." + fi + else + echo "[INFO] X11 Forwarding is configured as $__ORBIT_X11_FORWARDING_ENABLED in .container.yaml" + if [ "$__ORBIT_X11_FORWARDING_ENABLED" = "1" ]; then + echo "[INFO] To disable X11 forwarding, set __ORBIT_X11_FORWARDING_ENABLED=0 in .container.yaml" + else + echo "[INFO] To enable X11 forwarding, set __ORBIT_X11_FORWARDING_ENABLED=1 in .container.yaml" + fi + fi + + if [ "$__ORBIT_X11_FORWARDING_ENABLED" = "1" ]; then + configure_x11 + fi +} + +x11_cleanup() { + load_statefile_variable __ORBIT_TMP_XAUTH + if ! [ "$__ORBIT_TMP_XAUTH" = "null" ] && [ -f "$__ORBIT_TMP_XAUTH" ]; then + echo "[INFO] Removing temporary Orbit .xauth file $__ORBIT_TMP_XAUTH." + rm $__ORBIT_TMP_XAUTH + delete_statefile_variable __ORBIT_TMP_XAUTH + fi +} + #== # Main #== @@ -142,6 +247,10 @@ if ! command -v docker &> /dev/null; then exit 1 fi +if ! command -v yq &> /dev/null; then + install_yq +fi + # parse arguments mode="$1" profile_arg="$2" # Capture the second argument as the potential profile argument @@ -170,11 +279,13 @@ case $mode in start) echo "[INFO] Building the docker image and starting the container orbit-$container_profile in the background..." pushd ${SCRIPT_DIR} > /dev/null 2>&1 + # Determine if we want x11 forwarding enabled + x11_check # We have to build the base image as a separate step, # in case we are building a profile which depends # upon docker compose --file docker-compose.yaml --env-file .env.base build orbit-base - docker compose --file docker-compose.yaml $add_profiles $add_envs up --detach --build --remove-orphans + docker compose $add_yamls $add_profiles $add_envs up --detach --build --remove-orphans popd > /dev/null 2>&1 ;; enter) @@ -216,6 +327,7 @@ case $mode in echo "[INFO] Stopping the launched docker container orbit-$container_profile..." pushd ${SCRIPT_DIR} > /dev/null 2>&1 docker compose --file docker-compose.yaml $add_profiles $add_envs down + x11_cleanup popd > /dev/null 2>&1 ;; push) diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 658b7b2b3a..e6aa088b75 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -2,85 +2,76 @@ # be re-used between services to an # extension field # https://docs.docker.com/compose/compose-file/compose-file-v3/#extension-fields -x-default-orbit-volumes: - &default-orbit-volumes - # These volumes follow from this page - # https://docs.omniverse.nvidia.com/app_isaacsim/app_isaacsim/install_faq.html#save-isaac-sim-configs-on-local-disk - - type: volume - source: isaac-cache-kit - target: ${DOCKER_ISAACSIM_ROOT_PATH}/kit/cache - - type: volume - source: isaac-cache-ov - target: ${DOCKER_USER_HOME}/.cache/ov - - type: volume - source: isaac-cache-pip - target: ${DOCKER_USER_HOME}/.cache/pip - - type: volume - source: isaac-cache-gl - target: ${DOCKER_USER_HOME}/.cache/nvidia/GLCache - - type: volume - source: isaac-cache-compute - target: ${DOCKER_USER_HOME}/.nv/ComputeCache - - type: volume - source: isaac-logs - target: ${DOCKER_USER_HOME}/.nvidia-omniverse/logs - - type: volume - source: isaac-carb-logs - target: ${DOCKER_ISAACSIM_ROOT_PATH}/kit/logs/Kit/Isaac-Sim - - type: volume - source: isaac-data - target: ${DOCKER_USER_HOME}/.local/share/ov/data - - type: volume - source: isaac-docs - target: ${DOCKER_USER_HOME}/Documents - # These volumes allow X11 Forwarding - # We currently comment these out because they can - # cause bugs and warnings for people uninterested in - # X11 Forwarding from within the docker. We keep them - # as comments as a convenience for those seeking X11 - # forwarding until a scripted solution is developed - # - type: bind - # source: /tmp/.X11-unix - # target: /tmp/.X11-unix - # - type: bind - # source: ${HOME}/.Xauthority - # target: ${DOCKER_USER_HOME}/.Xauthority - # This overlay allows changes on the local files to - # be reflected within the container immediately - - type: bind - source: ../source - target: /workspace/orbit/source - - type: bind - source: ../docs - target: /workspace/orbit/docs - # The effect of these volumes is twofold: - # 1. Prevent root-owned files from flooding the _build and logs dir - # on the host machine - # 2. Preserve the artifacts in persistent volumes for later copying - # to the host machine - - type: volume - source: orbit-docs - target: /workspace/orbit/docs/_build - - type: volume - source: orbit-logs - target: /workspace/orbit/logs - - type: volume - source: orbit-data - target: /workspace/orbit/data_storage +x-default-orbit-volumes: &default-orbit-volumes + # These volumes follow from this page + # https://docs.omniverse.nvidia.com/app_isaacsim/app_isaacsim/install_faq.html#save-isaac-sim-configs-on-local-disk + - type: volume + source: isaac-cache-kit + target: ${DOCKER_ISAACSIM_ROOT_PATH}/kit/cache + - type: volume + source: isaac-cache-ov + target: ${DOCKER_USER_HOME}/.cache/ov + - type: volume + source: isaac-cache-pip + target: ${DOCKER_USER_HOME}/.cache/pip + - type: volume + source: isaac-cache-gl + target: ${DOCKER_USER_HOME}/.cache/nvidia/GLCache + - type: volume + source: isaac-cache-compute + target: ${DOCKER_USER_HOME}/.nv/ComputeCache + - type: volume + source: isaac-logs + target: ${DOCKER_USER_HOME}/.nvidia-omniverse/logs + - type: volume + source: isaac-carb-logs + target: ${DOCKER_ISAACSIM_ROOT_PATH}/kit/logs/Kit/Isaac-Sim + - type: volume + source: isaac-data + target: ${DOCKER_USER_HOME}/.local/share/ov/data + - type: volume + source: isaac-docs + target: ${DOCKER_USER_HOME}/Documents + # This overlay allows changes on the local files to + # be reflected within the container immediately + - type: bind + source: ../source + target: /workspace/orbit/source + - type: bind + source: ../docs + target: /workspace/orbit/docs + # The effect of these volumes is twofold: + # 1. Prevent root-owned files from flooding the _build and logs dir + # on the host machine + # 2. Preserve the artifacts in persistent volumes for later copying + # to the host machine + - type: volume + source: orbit-docs + target: /workspace/orbit/docs/_build + - type: volume + source: orbit-logs + target: /workspace/orbit/logs + - type: volume + source: orbit-data + target: /workspace/orbit/data_storage -x-default-orbit-deploy: - &default-orbit-deploy - resources: - reservations: - devices: - - driver: nvidia - count: all - capabilities: [ gpu ] +x-default-orbit-environment: &default-orbit-environment + - ISAACSIM_PATH=${DOCKER_ORBIT_PATH}/_isaac_sim + - ORBIT_PATH=${DOCKER_ORBIT_PATH} + - OMNI_KIT_ALLOW_ROOT=1 + +x-default-orbit-deploy: &default-orbit-deploy + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [ gpu ] services: # This service is the base Orbit image orbit-base: - profiles: ["base"] + profiles: [ "base" ] env_file: .env.base build: context: ../ @@ -92,13 +83,7 @@ services: - DOCKER_USER_HOME=${DOCKER_USER_HOME} image: orbit-base container_name: orbit-base - environment: - # We can't just define this in the .env file because shell envars take precedence - # https://docs.docker.com/compose/environment-variables/envvars-precedence/ - - ISAACSIM_PATH=${DOCKER_ORBIT_PATH}/_isaac_sim - - ORBIT_PATH=${DOCKER_ORBIT_PATH} - # This should also be enabled for X11 forwarding - # - DISPLAY=${DISPLAY} + environment: *default-orbit-environment volumes: *default-orbit-volumes network_mode: host deploy: *default-orbit-deploy @@ -110,7 +95,7 @@ services: # This service adds a ROS2 Humble # installation on top of the base image orbit-ros2: - profiles: ["ros2"] + profiles: [ "ros2" ] env_file: - .env.base - .env.ros2 @@ -118,16 +103,14 @@ services: context: ../ dockerfile: docker/Dockerfile.ros2 args: - # ROS2_APT_PACKAGE will default to NONE. This is to - # avoid a warning message when building only the base profile - # with the .env.base file + # ROS2_APT_PACKAGE will default to NONE. This is to + # avoid a warning message when building only the base profile + # with the .env.base file - ROS2_APT_PACKAGE=${ROS2_APT_PACKAGE:-NONE} - DOCKER_USER_HOME=${DOCKER_USER_HOME} image: orbit-ros2 container_name: orbit-ros2 - environment: - - ISAACSIM_PATH=${DOCKER_ORBIT_PATH}/_isaac_sim - - ORBIT_PATH=${DOCKER_ORBIT_PATH} + environment: *default-orbit-environment volumes: *default-orbit-volumes network_mode: host deploy: *default-orbit-deploy diff --git a/docker/x11.yaml b/docker/x11.yaml new file mode 100644 index 0000000000..bbd8f36ecc --- /dev/null +++ b/docker/x11.yaml @@ -0,0 +1,36 @@ +services: + orbit-base: + environment: + - DISPLAY + - TERM + - QT_X11_NO_MITSHM=1 + - XAUTHORITY=${__ORBIT_TMP_XAUTH} + volumes: + - type: bind + source: ${__ORBIT_TMP_XAUTH} + target: ${__ORBIT_TMP_XAUTH} + - type: bind + source: /tmp/.X11-unix + target: /tmp/.X11-unix + - type: bind + source: /etc/localtime + target: /etc/localtime + read_only: true + + orbit-ros2: + environment: + - DISPLAY + - TERM + - QT_X11_NO_MITSHM=1 + - XAUTHORITY=${__ORBIT_TMP_XAUTH} + volumes: + - type: bind + source: ${__ORBIT_TMP_XAUTH} + target: ${__ORBIT_TMP_XAUTH} + - type: bind + source: /tmp/.X11-unix + target: /tmp/.X11-unix + - type: bind + source: /etc/localtime + target: /etc/localtime + read_only: true