PR Review fixes:

- added support for multiple test subjects and parameterization in all tests

- reorganized file structure

- separate stats files for each aseg and aparc+DKT

- moving fixtures and common functions to common.py and conftest.py

quicktest_runner.yaml:

- removed check-output job
- added THIS_RUN_DIR to seperate different runs

.github/workflows/quicktest.yaml

@@ -1,90 +1,111 @@
-name: FastSurfer Singularity
+name: quicktest
+
+# File: quicktest.yaml
+# Author: Taha Abdullah
+# Created on: 2023-03-04
+# Functionality: This workflow runs some quick integration tests on FastSurfer commits. It checks out the new
+#                FastSurfer repo, sets up Python, builds a Singularity image, runs FastSurfer on sample MRI data, and
+#                runs pytest to check if the results are acceptable
+# Usage: This workflow is triggered on a push or pull-request to the dev and main branch of DeepMI/FastSurfer.
+#        It can also be triggered manually with workflow-dispatch
+
 
 on:
+#  pull_request:
   workflow_dispatch:
 
 jobs:
   # Checkout repo
   checkout:
-    runs-on: ci-gpu
+    runs-on: self-hosted
     steps:
     - uses: actions/checkout@v2
 
-  # Prepare job: Set up Python, Go, Singularity
+  # Prepare job: Set up Python, Go, Apptainer
   prepare-job:
-    runs-on: ci-gpu
+    runs-on: self-hosted
     needs: checkout
     steps:
     - name: Set up Python 3.10
       uses: actions/setup-python@v3
       with:
         python-version: "3.10"
-    - name: Set up Go 
-      uses: actions/setup-go@v5
-      with:
-        go-version: '^1.13.1' # The Go version to download (if necessary) and use.
+    - name: Install package
+      run: |
+        python -m pip install --progress-bar off --upgrade pip setuptools wheel
+        python -m pip install --progress-bar off .[test]
+#    - name: Set up Go
+#      uses: actions/setup-go@v5
+#      with:
+#        go-version: '^1.13.1' # The Go version to download (if necessary) and use.
     - name: Set up Singularity
       uses: eWaterCycle/setup-singularity@v7
       with:
-        singularity-version: 3.8.3
-        
-  # Build Docker Image and convert it to Singularity
-  build-singularity-image:
-    runs-on: ci-gpu
+        singularity-version: 3.8.7
+
+  # Build Docker Image and convert it to Apptainer
+  build-apptainer-image:
+    runs-on: self-hosted
     needs: prepare-job
     steps:
-    - name: Build Docker Image and convert to Singularity
+    - name: Build Docker Image and convert to Apptainer
       run: |
-        cd $RUNNER_SINGULARITY_IMGS
+        cd $RUNNER_FASTSURFER_IMGS
         FILE="fastsurfer-gpu.sif"
         if [ ! -f "$FILE" ]; then
         # If the file does not exist, build the file
           echo "SIF File does not exist. Building file."
-          PYTHONPATH=$PYTHONPATH
-          cd $PYTHONPATH
+          cd $FASTSURFER_HOME
           python3 Docker/build.py --device cuda --tag fastsurfer_gpu:cuda
-          cd $RUNNER_SINGULARITY_IMGS
-          singularity build --force fastsurfer-gpu.sif docker-daemon://fastsurfer_gpu:cuda
+          cd $RUNNER_FASTSURFER_IMGS
+          apptainer build --force fastsurfer-gpu.sif docker-daemon://fastsurfer_gpu:cuda
         else
           echo "File already exists"
-          cd $PYTHONPATH
+          cd $FASTSURFER_HOME
         fi
 
   # Run FastSurfer on MRI data
   run-fastsurfer:
-    runs-on: ci-gpu
-    needs: build-singularity-image
+    runs-on: self-hosted
+    needs: build-apptainer-image
     steps:
     - name: Run FastSurfer
       run: |
-        singularity exec --nv \
+        cd $RUNNER_FS_OUTPUT
+        # DIRECTORY="subjectX"
+        echo "pwd: $(pwd)"
+        # if [ -d "$DIRECTORY" ]; then
+        #   # if output already exists, delete it and run again
+        #   echo "Output already exists. Deleting output directory and running FastSurfer again."
+        #   rm -rf $DIRECTORY
+        # fi
+        apptainer exec --nv \
                       --no-home \
                       --bind $GITHUB_WORKSPACE:/fastsurfer-dev \
                       --env FASTSURFER_HOME=/fastsurfer-dev \
                       -B $RUNNER_FS_MRI_DATA:/data \
                       -B $RUNNER_FS_OUTPUT:/output \
                       -B $RUNNER_FS_LICENSE:/fs_license \
-                      $RUNNER_SINGULARITY_IMGS/fastsurfer-gpu.sif \
-                      /fastsurfer/run_fastsurfer.sh \
+                      $RUNNER_FASTSURFER_IMGS/fastsurfer-gpu.sif \
+                      /fastsurfer/brun_fastsurfer.sh \
                       --fs_license /fs_license/.license \
-                      --t1 /data/subjectx/orig.mgz \
-                      --sid subjectX --sd /output \
-                      --parallel --3T
+                      --subject_list /data/subject_list.txt \
+                      --sd /output \
+                      --parallel --3T \
+                      --parallel_subjects surf
 
-  # Test file existence
-  test-file-existence:
-    runs-on: ci-gpu
-    needs: run-fastsurfer
-    steps:
-    - name: Test File Existence
-      run: |
-        python3 test/quick_test/test_file_existence.py $RUNNER_FS_OUTPUT_FILES
-
-  # Test for errors in log files
-  test-error-messages:
-    runs-on: ci-gpu
-    needs: [run-fastsurfer, test-file-existence]
-    steps:
-    - name: Test Log Files For Error Messages
-      run: |
-        python3 test/quick_test/test_errors.py $RUNNER_FS_OUTPUT_LOGS
+  # Run pytest
+  run-pytest:
+      runs-on: self-hosted
+      needs: run-fastsurfer
+      steps:
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v3
+        with:
+          python-version: "3.10"
+      - name: Install package
+        run: |
+          python -m pip install --progress-bar off --upgrade pip setuptools wheel
+          python -m pip install --progress-bar off .[test]
+      - name : Run pytest
+        run: pytest test/quick_test

.github/workflows/quicktest_runner.yaml

@@ -0,0 +1,137 @@
+name: quicktest-runner
+
+# File: quicktest_runner.yaml
+# Author: Taha Abdullah
+# Created on: 2023-07-10
+# Functionality: This workflow runs FastSurfer on MRI data and runs pytest to check if the results are acceptable. It
+#  also checks if the FastSurfer environment and output already exist, and if not, it creates them.
+# Usage: This workflow is triggered on a pull request to the dev and main branch. It can also be triggered manually
+#  with workflow-dispatch.
+# Expected/Used Environment Variables:
+#  - MAMBAPATH: Path to the micromamba binary.
+#  - MAMBAROOT: Root path for micromamba.
+#  - RUNNER_FS_OUTPUT: Path to the directory where FastSurfer output is stored.
+#  - RUNNER_FS_MRI_DATA: Path to the directory where MRI data is stored.
+#  - FREESURFER_HOME: Path to the freesurfer directory.
+#  - FS_LICENSE: Path to the FreeSurfer license file.
+
+on:
+  pull_request:
+    branches:
+      - dev
+      - stable
+  workflow_dispatch:
+
+jobs:
+  # Checkout repo
+  checkout:
+    runs-on: self-hosted
+    steps:
+    - uses: actions/checkout@v2
+
+  # Create conda environment, install packages, and run Fastsurfer
+  run-fastsurfer:
+    runs-on: self-hosted
+    needs: checkout
+    steps:
+      # Check if the Environment Variables used in further steps are present
+    - name: Check Environment Variables
+      run: |
+        REQUIRED_ENV_VARS=(
+          "MAMBAPATH"
+          "MAMBAROOT"
+          "RUNNER_FS_OUTPUT"
+          "RUNNER_FS_MRI_DATA"
+          "FREESURFER_HOME"
+          "FS_LICENSE"
+        )
+        
+        for VAR_NAME in "${REQUIRED_ENV_VARS[@]}"; do
+          if [ -z "${!VAR_NAME}" ]; then
+            echo "Error: Required environment variable $VAR_NAME is not set"
+            exit 1
+          fi
+        done
+        
+        if [ ! -f "$FS_LICENSE" ]; then
+          echo "Error: FreeSurfer license file does not exist at $FS_LICENSE"
+          exit 1
+        fi
+        
+        if [ ! -d "$FREESURFER_HOME" ]; then
+          echo "Error: FreeSurfer installation directory does not exist at $FREESURFER_HOME"
+          exit 1
+        fi
+      # Set up Python using setup-python@v3 action
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v3
+      with:
+        python-version: "3.10"
+      # Check if FastSurfer environment already exists
+    - name: Check Environment
+      run: |
+        if [ ! -f "$MAMBAPATH" ]; then
+          echo "FastSurfer environment does not exist. Creating FastSurfer Environment..."
+          echo "HAS_MAMBA=true" >> $GITHUB_OUTPUT
+        else
+          echo "FastSurfer environment exists. Skipping FastSurfer Environment creation..."
+          echo "HAS_MAMBA=false" >> $GITHUB_OUTPUT
+        fi
+      id: check-environment
+      # Create FastSurfer environment if it does not exist
+    - name: Create FastSurfer Environment
+      uses: mamba-org/setup-micromamba@v1
+      with:
+        environment-file: env/fastsurfer.yml
+        micromamba-root-path: $MAMBAROOT
+        micromamba-binary-path: $MAMBAPATH
+        cache-downloads: true
+        cache-environment: true
+        init-shell: none
+      if: steps.check-environment.outputs.HAS_MAMBA == 'true'
+      # Set up cache for python packages
+    - name: Cache Python packages
+      uses: actions/cache@v3
+      with:
+        path: /home/.cache/pip
+        key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
+        restore-keys: |
+          ${{ runner.os }}-pip-
+      # Install required python packages
+    - name: Install package
+      run: |
+        python -m pip install --progress-bar off --upgrade pip \
+        setuptools wheel
+        python -m pip install --progress-bar off .[test]
+      # Run fastsurfer on list of subjects
+    - name: Run FastSurfer
+      run: |
+        echo "Running FastSurfer..."
+        export FREESURFER_HOME=/freesurfer
+        source $FREESURFER_HOME/SetUpFreeSurfer.sh
+        export FASTSURFER_HOME=$(pwd)
+        export THIS_RUN_OUTDIR=${GITHUB_SHA:0:7}
+        mkdir -p $SUBJECTS_DIR/$THIS_RUN_OUTDIR
+        ./brun_fastsurfer.sh --subject_list $RUNNER_FS_MRI_DATA/subjects_list.txt \
+                             --sd $SUBJECTS_DIR/$THIS_RUN_OUTDIR \
+                             --parallel --threads 4 --3T --parallel_subjects surf
+        export TEST_DIR=$THIS_RUN_OUTDIR
+
+  # Test fastsurfer output
+  run-pytest:
+    runs-on: self-hosted
+    needs: run-fastsurfer
+    steps:
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v3
+      with:
+        python-version: "3.10"
+    - name: Cache Python packages
+      uses: actions/cache@v3
+      with:
+        path: /home/.cache/pip
+        key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
+        restore-keys: |
+          ${{ runner.os }}-pip-
+    - name: Run pytest
+      run: python -m pytest test/quick_test

pyproject.toml

@@ -83,9 +83,13 @@ style = [
     'pydocstyle[toml]',
     'ruff',
 ]
+quicktest = [
+    'pytest>=8.2.2',
+]
 all = [
     'fastsurfer[doc]',
     'fastsurfer[style]',
+    'fastsurfer[quicktest]',
 ]
 full = [
     'fastsurfer[all]',

test/quick_test/common.py

@@ -0,0 +1,29 @@
+import os
+
+from logging import getLogger
+
+logger = getLogger(__name__)
+
+
+def load_test_subjects():
+    """
+    Load the test files from the given file path.
+
+    Returns:
+        test_subjects (list): List of subjects to test subjects.
+    """
+
+    subjects_dir = os.environ["SUBJECTS_DIR"]
+    subjects_list = os.environ["SUBJECTS_LIST"]
+
+    test_subjects = []
+
+    # Load the reference and test files
+    with open(os.path.join(subjects_dir, subjects_list), 'r') as file:
+        for line in file:
+            filename = line.strip()
+            logger.debug(filename)
+            # test_file = os.path.join(subjects_dir, filename)
+            test_subjects.append(filename)
+
+    return test_subjects

test/quick_test/conftest.py

@@ -0,0 +1,23 @@
+import os
+from pathlib import Path
+
+import pytest
+
+@pytest.fixture
+def subjects_dir():
+    return Path(os.environ["SUBJECTS_DIR"])
+
+
+@pytest.fixture
+def test_dir():
+    return Path(os.environ["TEST_DIR"])
+
+
+@pytest.fixture
+def reference_dir():
+    return Path(os.environ["REFERENCE_DIR"])
+
+
+@pytest.fixture
+def subjects_list():
+    return os.environ["SUBJECTS_LIST"]

test/quick_test/data/logfile.errors.yaml

@@ -0,0 +1,14 @@
+errors:
+ - "error"
+ - "error:"
+ - "exception"
+ - "traceback"
+
+whitelist:
+  - "without error"
+  - "not included"
+  - "distance"
+  - "correcting"
+  - "error="
+  - "rms error"
+  - "mcsrch error"