POSTPONED: VPN from within container

.github/workflows/pr-test.yaml

@@ -22,7 +22,6 @@ jobs:
       SITE_NAME: UKA
       PYTHONUNBUFFERED: 1
 
-
     steps:
       - name: Checkout repository (with submodules)
         uses: actions/checkout@v3
@@ -37,50 +36,60 @@ jobs:
           echo "VERSION=$VERSION"
           echo "version=$VERSION" >> $GITHUB_OUTPUT
 
-      - name: Build Docker image for real project (MEVIS)
-        run: |
-          chmod +x buildDockerImageAndStartupKits.sh
-          ./buildDockerImageAndStartupKits.sh -p application/provision/project_MEVIS_test.yml
+      - name: Build Docker image and startup kits for test project
+        run: ./buildDockerImageAndStartupKits.sh -p tests/provision/dummy_project_for_testing.yml -c tests/local_vpn/client_configs
 
-      - name: Show workspace path for MEVIS project
+      - name: Show workspace path for test project
         run: |
           echo "WORKSPACE_PATH: ${{ env.WORKSPACE_PATH }}"
-          find workspace -maxdepth 1 -type d -name "odelia_*_MEVIS_test" || echo "No workspace found"
+          find workspace -maxdepth 1 -type d -name "odelia_*_dummy_project_for_testing" || echo "No workspace found"
+
+      - name: Run integration test checking documentation on github
+        continue-on-error: false
+        run: |
+          ./runIntegrationTests.sh check_files_on_github
 
-      - name: Build Docker image and dummy startup kits
-        run: ./buildDockerImageAndStartupKits.sh -p tests/provision/dummy_project_for_testing.yml --use-docker-cache
+      - name: Run controller unit tests
+        continue-on-error: false
+        run: |
+          ./runIntegrationTests.sh run_unit_tests_controller
 
-      - name: Prepare dummy trainings
-        continue-on-error: true
+      - name: Run dummy training standalone
+        continue-on-error: false
         run: |
-          ./runTestsInDocker.sh prepare_dummy_trainings
-          echo "Dummy training project prepared"
+          ./runIntegrationTests.sh run_dummy_training_standalone
 
-      - name: Run dummy training
+      - name: Run dummy training in simulation mode
         continue-on-error: false
         run: |
-          ./runTestsInDocker.sh run_dummy_training
-          echo "Dummy training finished"
-          echo "=== Checking log output ==="
-          ls -lh workspace/*/prod_00/client_A/logs || echo "No logs found for dummy training"
+          ./runIntegrationTests.sh run_dummy_training_simulation_mode
 
-      - name: Run 3D CNN tests
+      - name: Run dummy training in proof-of-concept mode
         continue-on-error: false
         run: |
-          ./runTestsInDocker.sh run_3dcnn_tests
-          echo "3D CNN tests check finished"
-          echo "=== Checking synthetic log output ==="
-          ls -lh workspace/*/prod_00/client_A/logs || echo "No logs found for 3D CNN tests"
+          ./runIntegrationTests.sh run_dummy_training_poc_mode
 
-      - name: Run Unit Tests inside Docker
-        continue-on-error: true
+      - name: Run 3DCNN training in simulation mode
+        continue-on-error: false
         run: |
-          ./runTestsInDocker.sh run_tests
-          echo "=== [LOG CHECK] ==="
-          docker logs $(docker ps -a -q --latest) | grep -i "error" && echo "Error found in logs" || echo "No error found"
+          ./runIntegrationTests.sh run_3dcnn_simulation_mode
 
-      - name: Cleanup training artifacts
-        continue-on-error: true
+      - name: Run integration test creating startup kits
+        continue-on-error: false
+        run: |
+          ./runIntegrationTests.sh create_startup_kits
+
+      - name: Run intergration test listing licenses
+        continue-on-error: false
+        run: |
+          ./runIntegrationTests.sh run_list_licenses
+
+      - name: Run integration test Docker GPU preflight check
+        continue-on-error: false
+        run: |
+          ./runIntegrationTests.sh run_docker_gpu_preflight_check
+
+      - name: Run integration test Data access preflight check
+        continue-on-error: false
         run: |
-          ./runTestsInDocker.sh cleanup
-          echo "Cleanup finished"
+          ./runIntegrationTests.sh run_data_access_preflight_check

_buildStartupKits.sh

@@ -2,25 +2,32 @@
 
 set -euo pipefail
 
-if [ "$#" -ne 2 ]; then
-    echo "Usage: _buildStartupKits.sh SWARM_PROJECT.yml VERSION_STRING"
+if [ "$#" -lt 3 ]; then
+    echo "Usage: _buildStartupKits.sh SWARM_PROJECT.yml VERSION_STRING CONTAINER_NAME [VPN_CREDENTIALS_DIR]"
     exit 1
 fi
 
 PROJECT_YML=$1
 VERSION=$2
+CONTAINER_NAME=$3
+MOUNT_VPN_CREDENTIALS_DIR=""
+if [ "$#" -eq 4 ]; then
+    MOUNT_VPN_CREDENTIALS_DIR="-v $4:/vpn_credentials/"
+fi
 
 sed -i 's#__REPLACED_BY_CURRENT_VERSION_NUMBER_WHEN_BUILDING_STARTUP_KITS__#'$VERSION'#' $PROJECT_YML
-echo "Building startup kits for project $PROJECT_YML with version $VERSION"
+
+ARGUMENTS="$PROJECT_YML $VERSION"
+
+echo "Building startup kits: $ARGUMENTS"
 docker run --rm \
   -u $(id -u):$(id -g) \
   -v /etc/passwd:/etc/passwd \
   -v /etc/group:/etc/group \
   -v ./:/workspace/ \
+  $MOUNT_VPN_CREDENTIALS_DIR \
   -w /workspace/ \
-  -e PROJECT_YML=$PROJECT_YML \
-  -e VERSION=$VERSION \
-  jefftud/odelia:$VERSION \
-  /bin/bash -c "nvflare provision -p \$PROJECT_YML && ./_generateStartupKitArchives.sh \$PROJECT_YML \$VERSION"|| { echo "Docker run failed"; exit 1; }
+  $CONTAINER_NAME \
+  /bin/bash -c "nvflare provision -p $PROJECT_YML && ./_generateStartupKitArchives.sh $ARGUMENTS"|| { echo "Docker run failed"; exit 1; }
 
 sed -i 's#'$VERSION'#__REPLACED_BY_CURRENT_VERSION_NUMBER_WHEN_BUILDING_STARTUP_KITS__#' $PROJECT_YML

_generateStartupKitArchives.sh

@@ -7,7 +7,14 @@ TARGET_FOLDER=`ls -d $OUTPUT_FOLDER/prod_* | tail -n 1`
 LONG_VERSION=$2
 
 cd $TARGET_FOLDER
+
 for startupkit in `ls .`; do
+    VPN_CREDENTIALS_FILE=/vpn_credentials/${startupkit}_client.ovpn
+    if [[ -f $VPN_CREDENTIALS_FILE ]]; then
+        cp $VPN_CREDENTIALS_FILE ${startupkit}/startup/vpn_client.ovpn
+    else
+        echo "$VPN_CREDENTIALS_FILE does not exist, omitting VPN credentials for ${startupkit} in startup kit"
+    fi
     zip -rq ${startupkit}_$LONG_VERSION.zip $startupkit
     echo "Generated startup kit $TARGET_FOLDER/${startupkit}_$LONG_VERSION.zip"
 done

assets/readme/README.developer.md

@@ -24,22 +24,24 @@ The project description specifies the swarm nodes etc. to be used for a swarm tr
    kits, running local trainings in the startup kit), you can manually push the image to DockerHub, provided you have
    the necessary rights. Make sure you are not re-using a version number for this purpose.
 
-## Running Local Tests
+## Running Tests
 
    ```bash
-   ./runTestsInDocker.sh
+   ./runIntegrationTests.sh
    ```
 
 You should see
 
 1. several expected errors and warnings printed from unit tests that should succeed overall, and a coverage report
-2. output of a successful simulation run with two nodes
-3. output of a successful proof-of-concept run run with two nodes
-4. output of a set of startup kits being generated
-5. output of a dummy training run using one of the startup kits
-6. TODO update this to what the tests output now
+2. output of a successful simulation run of a dummy training with two nodes
+3. output of a successful proof-of-concept run of a dummy training with two nodes
+4. output of a successful simulation run of a 3D CNN training using synthetic data with two nodes
+5. output of a set of startup kits being generated
+6. output of a Docker/GPU preflight check using one of the startup kits
+7. output of a data access preflight check using one of the startup kits
+8. output of a dummy training run in a swarm consisting of one server and two client nodes
 
-Optionally, uncomment running NVFlare unit tests in `_runTestsInsideDocker.sh`.
+Optionally, uncomment running NVFlare unit tests.
 
 ## Distributing Startup Kits
 
@@ -93,3 +95,7 @@ export CONFIG=original
    run in the swarm
 3. Use the local tests to check if the code is swarm-ready
 4. TODO more detailed instructions
+
+## Continuous Integration
+
+Tests to be executed after pushing to github are defined in `.github/workflows/pr-test.yaml`.

assets/readme/README.operator.md

@@ -19,17 +19,25 @@ For example, add the following line (replace `<IP>` with the server's actual IP
 <IP>    dl3.tud.de dl3
 ```
 
+TODO describe this in participant REAME if needed
+
 ## Create Startup Kits
 
 ### Via Script (recommended)
 
 1. Use, e.g., the file `application/provision/project_MEVIS_test.yml`, adapt as needed (network protocol etc.)
-2. Call `buildStartupKits.sh /path/to/project_configuration.yml` to build the startup kits
+2. Call `buildDockerImageAndStartupKits.sh -p /path/to/project_configuration.yml -c /path/to/directory/with/VPN/credentials` to build the Docker image and the startup kits
+  - swarm nodes (admin, server, clients) are configured in `project_configuration.yml`
+  - the directory with VPN credentials should contain one `.ovpn` file per node
+  - use `-c tests/local_vpn/client_configs/` to build startup kits for the integration tests
 3. Startup kits are generated to `workspace/<name configured in the .yml>/prod_00/`
-4. Deploy startup kits to the respective server/clients
+4. Deploy startup kits to the respective server/client operators
+5. Push the Docker image to the registry
 
 ### Via the Dashboard (not recommended)
 
+Build the Docker image as described above.
+
 ```bash
 docker run -d --rm \
      --ipc=host -p 8443:8443 \
@@ -69,14 +77,14 @@ Access the dashboard at `https://localhost:8443` log in with the admin credentia
 2. Client Sites > approve client sites
 3. Project Home > freeze project
 
-## Download startup kits
+#### Download startup kits
 
 After setting up the project admin configuration, server and clients can download their startup kits. Store the
 passwords somewhere, they are only displayed once (or you can download them again).
 
 ## Starting a Swarm Training
 
-1. Connect the *server* host to the VPN as described above.
+1. Connect the *server* host to the VPN as described above. (TODO update documentation, this step is not needed if the Docker container handles the VPN connection.)
 2. Start the *server* startup kit using the respective `startup/docker.sh` script with the option to start the server
 3. Provide the *client* startup kits to the swarm participants (be aware that email providers or other channels may
    prevent encrypted archives)

buildDockerImageAndStartupKits.sh

@@ -13,20 +13,20 @@ DOCKER_BUILD_ARGS="--no-cache --progress=plain";
 while [[ "$#" -gt 0 ]]; do
     case $1 in
         -p)                  PROJECT_FILE="$2"; shift ;;
+        -c)                  VPN_CREDENTIALS_DIR="$2"; shift ;;
         --use-docker-cache)  DOCKER_BUILD_ARGS="";;
         *) echo "Unknown parameter passed: $1"; exit 1 ;;
     esac
     shift
 done
 
-if [ -z "$PROJECT_FILE" ]; then
-    echo "Usage: buildDockerImageAndStartupKits.sh -p <swarm_project.yml> [--use-docker-cache]"
+if [[ -z "$PROJECT_FILE" || -z "$VPN_CREDENTIALS_DIR" ]]; then
+    echo "Usage: buildDockerImageAndStartupKits.sh -p <swarm_project.yml> -c <VPN credentials directory> [--use-docker-cache]"
     exit 1
 fi
 
 VERSION=`./getVersionNumber.sh`
-DOCKER_IMAGE=jefftud/odelia:$VERSION
-
+CONTAINER_VERSION_ID=`git rev-parse --short HEAD`
 
 # prepare clean version of source code repository clone for building Docker image
 
@@ -41,14 +41,15 @@ git clean -x -q -f .
 cd ../..
 rm .git -rf
 chmod a+rX . -R
-sed -i 's#__REPLACED_BY_CURRENT_VERSION_NUMBER_WHEN_BUILDING_DOCKER_IMAGE__#'$VERSION'#' docker_config/master_template.yml
-cd $CWD
 
+# replacements in copy of source code
+sed -i 's#__REPLACED_BY_CURRENT_VERSION_NUMBER_WHEN_BUILDING_DOCKER_IMAGE__#'$VERSION'#' docker_config/master_template.yml
+sed -i 's#__REPLACED_BY_CONTAINER_VERSION_IDENTIFIER_WHEN_BUILDING_DOCKER_IMAGE__#'$CONTAINER_VERSION_ID'#' docker_config/master_template.yml
 
 # prepare pre-trained model weights for being included in Docker image
 
-MODEL_WEIGHTS_FILE='docker_config/torch_home_cache/hub/checkpoints/dinov2_vits14_pretrain.pth'
-MODEL_LICENSE_FILE='docker_config/torch_home_cache/hub/facebookresearch_dinov2_main/LICENSE'
+MODEL_WEIGHTS_FILE=$CWD'/docker_config/torch_home_cache/hub/checkpoints/dinov2_vits14_pretrain.pth'
+MODEL_LICENSE_FILE=$CWD'/docker_config/torch_home_cache/hub/facebookresearch_dinov2_main/LICENSE'
 if [[ ! -f $MODEL_WEIGHTS_FILE || ! -f $MODEL_LICENSE_FILE ]]; then
     echo "Pre-trained model not available. Attempting download"
     HUBDIR=$(dirname $(dirname $MODEL_LICENSE_FILE))
@@ -61,22 +62,26 @@ if [[ ! -f $MODEL_WEIGHTS_FILE || ! -f $MODEL_LICENSE_FILE ]]; then
 fi
 
 if echo 2e405cee1bad14912278296d4f42e993 $MODEL_WEIGHTS_FILE | md5sum --check - && echo 153d2db1c329326a2d9f881317ea942e $MODEL_LICENSE_FILE | md5sum --check -; then
-    cp -r ./docker_config/torch_home_cache $CLEAN_SOURCE_DIR/torch_home_cache
+    cp -r $CWD/docker_config/torch_home_cache $CLEAN_SOURCE_DIR/torch_home_cache
 else
     exit 1
 fi
 chmod a+rX $CLEAN_SOURCE_DIR/torch_home_cache -R
 
+cd $CWD
 
 # build and print follow-up steps
+CONTAINER_NAME=`grep "      docker_image: " $PROJECT_FILE | sed 's/      docker_image: //' | sed 's#__REPLACED_BY_CURRENT_VERSION_NUMBER_WHEN_BUILDING_STARTUP_KITS__#'$VERSION'#'`
+echo $CONTAINER_NAME
 
-docker build $DOCKER_BUILD_ARGS -t $DOCKER_IMAGE $CLEAN_SOURCE_DIR -f docker_config/Dockerfile_ODELIA
+docker build $DOCKER_BUILD_ARGS -t $CONTAINER_NAME $CLEAN_SOURCE_DIR -f docker_config/Dockerfile_ODELIA
 
-echo "Docker image $DOCKER_IMAGE built successfully"
-echo "./_buildStartupKits.sh $PROJECT_FILE $VERSION"
-./_buildStartupKits.sh $PROJECT_FILE $VERSION
+echo "Docker image $CONTAINER_NAME built successfully"
+echo "./_buildStartupKits.sh $PROJECT_FILE $VERSION $CONTAINER_NAME"
+VPN_CREDENTIALS_DIR=$(realpath $VPN_CREDENTIALS_DIR)
+./_buildStartupKits.sh $PROJECT_FILE $VERSION $CONTAINER_NAME $VPN_CREDENTIALS_DIR
 echo "Startup kits built successfully"
 
 rm -rf $CLEAN_SOURCE_DIR
 
-echo "If you wish, manually push $DOCKER_IMAGE now"
+echo "If you wish, manually push $CONTAINER_NAME now"