Merge branch 'compress-symbols-separately' into 'master'

Compress symbols separately See merge request OpenMW/openmw!4362
OpenMW · Sep 9, 2024 · d643970 · d643970
2 parents 7a6fa45 + 6723d6c
commit d643970
Show file tree

Hide file tree

Showing 2 changed files with 104 additions and 9 deletions.
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -533,6 +533,45 @@ macOS14_Xcode15_arm64:
   variables:
     CCACHE_SIZE: 3G
 
+.Compress_And_Upload_Symbols_Base:
+  extends: .Ubuntu_Image
+  stage: build
+  variables:
+    GIT_STRATEGY: none
+  script:
+    - apt-get update
+    - apt-get install -y curl gcab unzip
+    - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o awscli-exe-linux-x86_64.zip
+    - unzip -d awscli-exe-linux-x86_64 awscli-exe-linux-x86_64.zip
+    - pushd awscli-exe-linux-x86_64
+    - ./aws/install
+    - popd
+    - aws --version
+    - unzip -d sym_store *sym_store.zip
+    - shopt -s globstar
+    - |
+      for file in sym_store/**/*.exe; do
+        if [[ -f "$file" ]]; then
+          gcab --create --zip --nopath "${file%.exe}.ex_" "$file"
+        fi
+      done
+    - |
+      for file in sym_store/**/*.dll; do
+        if [[ -f "$file" ]]; then
+          gcab --create --zip --nopath "${file%.dll}.dl_" "$file"
+        fi
+      done
+    - |
+      for file in sym_store/**/*.pdb; do
+        if [[ -f "$file" ]]; then
+          gcab --create --zip --nopath "${file%.pdb}.pd_" "$file"
+        fi
+      done
+    - |
+      if [[ -v AWS_ACCESS_KEY_ID ]]; then
+          aws --endpoint-url https://rgw.ctrl-c.liu.se s3 cp --recursive --exclude '*' --include '*.ex_' --include '*.dl_' --include '*.pd_' sym_store s3://openmw-sym
+      fi
+
 .Windows_Ninja_Base:
   tags:
     - saas-windows-medium-amd64
@@ -590,10 +629,7 @@ macOS14_Xcode15_arm64:
           aws --endpoint-url https://rgw.ctrl-c.liu.se s3 cp "..\..\$(Make-SafeFileName("OpenMW_MSVC2022_64_${config}_${CI_COMMIT_REF_NAME}_${CI_JOB_ID}_symbols.zip"))" s3://openmw-artifacts/${artifactDirectory}
         }
         Push-Location ..
-        ..\CI\Store-Symbols.ps1
-        if (Test-Path env:AWS_ACCESS_KEY_ID) {
-          aws --endpoint-url https://rgw.ctrl-c.liu.se s3 cp --recursive --exclude * --include *.ex_ --include *.dl_ --include *.pd_ .\SymStore s3://openmw-sym
-        }
+        ..\CI\Store-Symbols.ps1 -SkipCompress
         7z a -tzip "..\$(Make-SafeFileName("OpenMW_MSVC2022_64_${config}_${CI_COMMIT_REF_NAME}_${CI_JOB_ID}_sym_store.zip"))" '.\SymStore\*' $config\CI-ID.txt
         Pop-Location
         Get-ChildItem -Recurse *.pdb | Remove-Item
@@ -631,19 +667,40 @@ macOS14_Xcode15_arm64:
   variables:
     config: "Release"
 
+.Windows_Compress_And_Upload_Symbols_Ninja_Release:
+  extends:
+    - .Compress_And_Upload_Symbols_Base
+  needs:
+    - job: "Windows_Ninja_Release"
+      artifacts: true
+
 .Windows_Ninja_Release_MultiView:
   extends:
     - .Windows_Ninja_Base
   variables:
     multiview: "-M"
     config: "Release"
 
+.Windows_Compress_And_Upload_Symbols_Ninja_Release_MultiView:
+  extends:
+    - .Compress_And_Upload_Symbols_Base
+  needs:
+    - job: "Windows_Ninja_Release_MultiView"
+      artifacts: true
+
 .Windows_Ninja_Debug:
   extends:
     - .Windows_Ninja_Base
   variables:
     config: "Debug"
 
+.Windows_Compress_And_Upload_Symbols_Ninja_Debug:
+  extends:
+    - .Compress_And_Upload_Symbols_Base
+  needs:
+    - job: "Windows_Ninja_Debug"
+      artifacts: true
+
 .Windows_Ninja_RelWithDebInfo:
   extends:
     - .Windows_Ninja_Base
@@ -652,6 +709,13 @@ macOS14_Xcode15_arm64:
     # Gitlab can't successfully execute following binaries due to unknown reason
     # executables: "components-tests.exe,openmw-tests.exe,openmw-cs-tests.exe,openmw_detournavigator_navmeshtilescache_benchmark.exe"
 
+.Windows_Compress_And_Upload_Symbols_Ninja_RelWithDebInfo:
+  extends:
+    - .Compress_And_Upload_Symbols_Base
+  needs:
+    - job: "Windows_Ninja_RelWithDebInfo"
+      artifacts: true
+
 .Windows_Ninja_CacheInit:
   # currently, Windows jobs for all configs share the same cache key as we only cache the dependencies
   extends:
@@ -711,10 +775,7 @@ macOS14_Xcode15_arm64:
           aws --endpoint-url https://rgw.ctrl-c.liu.se s3 cp "..\..\$(Make-SafeFileName("OpenMW_MSVC2022_64_${config}_${CI_COMMIT_REF_NAME}_${CI_JOB_ID}_symbols.zip"))" s3://openmw-artifacts/${artifactDirectory}
         }
         Push-Location ..
-        ..\CI\Store-Symbols.ps1
-        if (Test-Path env:AWS_ACCESS_KEY_ID) {
-          aws --endpoint-url https://rgw.ctrl-c.liu.se s3 cp --recursive --exclude * --include *.ex_ --include *.dl_ --include *.pd_ .\SymStore s3://openmw-sym
-        }
+        ..\CI\Store-Symbols.ps1 -SkipCompress
         7z a -tzip "..\$(Make-SafeFileName("OpenMW_MSVC2022_64_${config}_${CI_COMMIT_REF_NAME}_${CI_JOB_ID}_sym_store.zip"))" '.\SymStore\*' $config\CI-ID.txt
         Pop-Location
         Get-ChildItem -Recurse *.pdb | Remove-Item
@@ -751,12 +812,26 @@ macOS14_Xcode15_arm64:
   variables:
     config: "Release"
 
+.Windows_Compress_And_Upload_Symbols_MSBuild_Release:
+  extends:
+    - .Compress_And_Upload_Symbols_Base
+  needs:
+    - job: "Windows_MSBuild_Release"
+      artifacts: true
+
 .Windows_MSBuild_Debug:
   extends:
     - .Windows_MSBuild_Base
   variables:
     config: "Debug"
 
+.Windows_Compress_And_Upload_Symbols_MSBuild_Debug:
+  extends:
+    - .Compress_And_Upload_Symbols_Base
+  needs:
+    - job: "Windows_MSBuild_Debug"
+      artifacts: true
+
 Windows_MSBuild_RelWithDebInfo:
   extends:
     - .Windows_MSBuild_Base
@@ -769,6 +844,17 @@ Windows_MSBuild_RelWithDebInfo:
     # run this for both pushes and schedules so 'latest successful pipeline for branch' always includes it
     - if: $CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event" || $CI_PIPELINE_SOURCE == "schedule"
 
+Windows_Compress_And_Upload_Symbols_MSBuild_RelWithDebInfo:
+  extends:
+    - .Compress_And_Upload_Symbols_Base
+  needs:
+    - job: "Windows_MSBuild_RelWithDebInfo"
+      artifacts: true
+  # temporarily enabled while we're linking the above on the downloads page
+  rules:
+    # run this for both pushes and schedules so 'latest successful pipeline for branch' always includes it
+    - if: $CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event" || $CI_PIPELINE_SOURCE == "schedule"
+
 Windows_MSBuild_CacheInit:
   # currently, Windows jobs for all configs share the same cache key as we only cache the dependencies
   extends:

diff --git a/CI/Store-Symbols.ps1 b/CI/Store-Symbols.ps1
@@ -1,3 +1,7 @@
+param (
+    [switch] $SkipCompress
+)
+
 $ErrorActionPreference = "Stop"
 
 if (-Not (Test-Path CMakeCache.txt))
@@ -67,7 +71,12 @@ $artifacts = $artifacts | Where-Object { Test-Path $_ }
 
 Write-Output "Storing symbols..."
 
-symstore-venv\Scripts\symstore --compress --skip-published .\SymStore @artifacts
+$optionalArgs = @()
+if (-not $SkipCompress) {
+    $optionalArgs += "--compress"
+}
+
+symstore-venv\Scripts\symstore $optionalArgs --skip-published .\SymStore @artifacts
 if ($LASTEXITCODE -ne 0) {
     Write-Error "Command exited with code $LASTEXITCODE"
 }