CI: Test schemes (#800)

Let's try to separate on-device tests from the core suite (with proper
retry, timeout, etc.).

Author: pblazej

.github/workflows/ci.yaml

@@ -16,7 +16,9 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  TESTS: "FunctionTests SDKTests RoomTests PublishDataTests PublishBufferCapturerTests VideoViewTests TrackTests"
+  TEST_TARGETS: "LiveKitCoreTests LiveKitObjCTests"
+  TEST_TIMEOUT_MINUTES: 10
+  TEST_RETRY_ATTEMPTS: 3
 
 jobs:
   build-and-test:
@@ -43,11 +45,9 @@ jobs:
           - os: macos-15
             xcode: 16.4
             platform: "iOS Simulator,name=iPhone 16 Pro,OS=18.5"
-            symbol-graph: true
           - os: macos-15
             xcode: 16.4
             platform: "macOS"
-            symbol-graph: true
           - os: macos-15
             xcode: 16.4
             platform: "macOS,variant=Mac Catalyst"
@@ -62,9 +62,19 @@ jobs:
           - os: macos-26
             xcode: latest
             platform: "iOS Simulator,name=iPhone 17 Pro,OS=26.0"
+            symbol-graph: true
+          - os: macos-26
+            xcode: latest
+            platform: "macOS"
+            symbol-graph: true
           - os: macos-26
             xcode: latest
             platform: "macOS"
+            asan: true
+          - os: macos-26
+            xcode: latest
+            platform: "macOS"
+            tsan: true
           - os: macos-26
             xcode: latest
             platform: "macOS,variant=Mac Catalyst"
@@ -103,20 +113,36 @@ jobs:
         run: |
           set -o pipefail && xcodebuild build-for-testing \
             -scheme LiveKit \
-            -destination 'platform=${{ matrix.platform }}' | xcbeautify --renderer github-actions
+            -destination 'platform=${{ matrix.platform }}' \
+            -enableAddressSanitizer ${{ matrix.asan == true && 'YES' || 'NO' }} \
+            -enableThreadSanitizer ${{ matrix.tsan == true && 'YES' || 'NO' }} \
+            | xcbeautify --renderer github-actions
 
-      # Run all tests in parallel instead of sequentially
       - name: Run Tests
-        run: |
-          IFS=' ' read -r -a tests <<< "${{ env.TESTS }}"
-          for test in "${tests[@]}"; do
-            echo "::group::Running test $test"
-            set -o pipefail && xcodebuild test-without-building \
-              -scheme LiveKit \
-              -destination 'platform=${{ matrix.platform }}' \
-              -only-testing:LiveKitTests/$test | xcbeautify --renderer github-actions
-            echo "::endgroup::"
-          done
+        uses: nick-fields/retry@v3
+        with:
+          timeout_minutes: ${{ env.TEST_TIMEOUT_MINUTES }}
+          max_attempts: ${{ env.TEST_RETRY_ATTEMPTS }}
+          command: |
+            set -euo pipefail
+            IFS=' ' read -r -a tests <<< "${{ env.TEST_TARGETS }}"
+
+            for test in "${tests[@]}"; do
+              echo "::group::$test"
+              if ! xcodebuild test-without-building \
+                -scheme LiveKit \
+                -destination 'platform=${{ matrix.platform }}' \
+                -enableAddressSanitizer ${{ matrix.asan == true && 'YES' || 'NO' }} \
+                -enableThreadSanitizer ${{ matrix.tsan == true && 'YES' || 'NO' }} \
+                -only-testing:$test \
+                -parallel-testing-enabled YES \
+                | xcbeautify --renderer github-actions
+              then
+                echo "::error::Test failed: $test"
+                exit 1
+              fi
+              echo "::endgroup::"
+            done
 
       - name: Build for Release
         if: ${{ matrix.symbol-graph }}

LiveKit.xctestplan

@@ -53,18 +53,33 @@ let package = Package(
                 .enableExperimentalFeature("AccessLevelOnImport"),
             ]
         ),
-        .testTarget(
-            name: "LiveKitTests",
+        .target(
+            name: "LiveKitTestSupport",
             dependencies: [
                 "LiveKit",
                 .product(name: "JWTKit", package: "jwt-kit"),
+            ],
+            path: "Tests/LiveKitTestSupport"
+        ),
+        .testTarget(
+            name: "LiveKitCoreTests",
+            dependencies: [
+                "LiveKit",
+                "LiveKitTestSupport",
             ]
         ),
         .testTarget(
-            name: "LiveKitTestsObjC",
+            name: "LiveKitAudioTests",
             dependencies: [
                 "LiveKit",
-                .product(name: "JWTKit", package: "jwt-kit"),
+                "LiveKitTestSupport",
+            ]
+        ),
+        .testTarget(
+            name: "LiveKitObjCTests",
+            dependencies: [
+                "LiveKit",
+                "LiveKitTestSupport",
             ]
         ),
     ],

Package.swift

@@ -54,18 +54,33 @@ let package = Package(
                 .enableExperimentalFeature("AccessLevelOnImport"),
             ]
         ),
-        .testTarget(
-            name: "LiveKitTests",
+        .target(
+            name: "LiveKitTestSupport",
             dependencies: [
                 "LiveKit",
                 .product(name: "JWTKit", package: "jwt-kit"),
+            ],
+            path: "Tests/LiveKitTestSupport"
+        ),
+        .testTarget(
+            name: "LiveKitCoreTests",
+            dependencies: [
+                "LiveKit",
+                "LiveKitTestSupport",
             ]
         ),
         .testTarget(
-            name: "LiveKitTestsObjC",
+            name: "LiveKitAudioTests",
             dependencies: [
                 "LiveKit",
-                .product(name: "JWTKit", package: "jwt-kit"),
+                "LiveKitTestSupport",
+            ]
+        ),
+        .testTarget(
+            name: "LiveKitObjCTests",
+            dependencies: [
+                "LiveKit",
+                "LiveKitTestSupport",
             ]
         ),
     ],

[email protected]

@@ -16,7 +16,7 @@
 
 @preconcurrency import AVFoundation
 @testable import LiveKit
-import XCTest
+import LiveKitTestSupport
 
 class AudioEngineAvailabilityTests: LKTestCase {
     // Check if audio engine will stop when availability is set to .none,

…ests/Audio/AudioEngineAvailability.swift …AudioTests/AudioEngineAvailability.swiftTests/LiveKitTests/Audio/AudioEngineAvailability.swift renamed to Tests/LiveKitAudioTests/AudioEngineAvailability.swift Tests/LiveKitTests/Audio/AudioEngineAvailability.swift renamed to Tests/LiveKitAudioTests/AudioEngineAvailability.swift

@@ -16,7 +16,7 @@
 
 @preconcurrency import AVFoundation
 @testable import LiveKit
-import XCTest
+import LiveKitTestSupport
 
 final class TestEngineObserver: AudioEngineObserver, @unchecked Sendable {
     var next: (any LiveKit.AudioEngineObserver)?

…KitTests/Audio/AudioEngineObserver.swift …eKitAudioTests/AudioEngineObserver.swiftTests/LiveKitTests/Audio/AudioEngineObserver.swift renamed to Tests/LiveKitAudioTests/AudioEngineObserver.swift Tests/LiveKitTests/Audio/AudioEngineObserver.swift renamed to Tests/LiveKitAudioTests/AudioEngineObserver.swift

@@ -16,7 +16,7 @@
 
 @preconcurrency import AVFoundation
 @testable import LiveKit
-import XCTest
+import LiveKitTestSupport
 
 class AudioEnginePermissionTests: LKTestCase {
     #if os(iOS) || os(visionOS) || os(tvOS)

…tTests/Audio/AudioEnginePermission.swift …itAudioTests/AudioEnginePermission.swiftTests/LiveKitTests/Audio/AudioEnginePermission.swift renamed to Tests/LiveKitAudioTests/AudioEnginePermission.swift Tests/LiveKitTests/Audio/AudioEnginePermission.swift renamed to Tests/LiveKitAudioTests/AudioEnginePermission.swift

@@ -16,9 +16,9 @@
 
 @preconcurrency import AVFoundation
 @testable import LiveKit
+import LiveKitTestSupport
 import LiveKitWebRTC
 import LKObjCHelpers
-import XCTest
 
 class AudioEngineTests: LKTestCase, @unchecked Sendable {
     override func tearDown() async throws {}

Tests/LiveKitTests/AudioEngineTests.swift …LiveKitAudioTests/AudioEngineTests.swiftTests/LiveKitTests/AudioEngineTests.swift renamed to Tests/LiveKitAudioTests/AudioEngineTests.swift Tests/LiveKitTests/AudioEngineTests.swift renamed to Tests/LiveKitAudioTests/AudioEngineTests.swift

@@ -16,8 +16,8 @@
 
 @preconcurrency import AVFoundation
 @testable import LiveKit
+import LiveKitTestSupport
 import LiveKitWebRTC
-import XCTest
 
 class AudioManagerTests: LKTestCase {
     // Test legacy audio device module's startLocalRecording().

…veKitTests/Audio/AudioManagerTests.swift …iveKitAudioTests/AudioManagerTests.swiftTests/LiveKitTests/Audio/AudioManagerTests.swift renamed to Tests/LiveKitAudioTests/AudioManagerTests.swift Tests/LiveKitTests/Audio/AudioManagerTests.swift renamed to Tests/LiveKitAudioTests/AudioManagerTests.swift

@@ -20,7 +20,7 @@ import Foundation
 import LiveKitWebRTC
 
 @testable import LiveKit
-import XCTest
+import LiveKitTestSupport
 
 class AudioProcessingTests: LKTestCase, AudioCustomProcessingDelegate, @unchecked Sendable {
     var _initSampleRate: Double = 0.0