feat(macOS): Capture audio on macOS using Tap API

.gitignore

@@ -56,3 +56,6 @@ package-lock.json
 # Python
 *.pyc
 venv/
+
+
+.cache/

cmake/compile_definitions/macos.cmake

@@ -24,6 +24,9 @@ list(APPEND SUNSHINE_EXTERNAL_LIBRARIES
         ${CORE_MEDIA_LIBRARY}
         ${CORE_VIDEO_LIBRARY}
         ${FOUNDATION_LIBRARY}
+        ${AUDIO_TOOLBOX_LIBRARY}
+        ${AUDIO_UNIT_LIBRARY}
+        ${CORE_AUDIO_LIBRARY}
         ${VIDEO_TOOLBOX_LIBRARY})
 
 set(APPLE_PLIST_FILE "${SUNSHINE_SOURCE_ASSETS_DIR}/macos/assets/Info.plist")
@@ -33,7 +36,7 @@ set(SUNSHINE_TRAY 0)
 
 set(PLATFORM_TARGET_FILES
         "${CMAKE_SOURCE_DIR}/src/platform/macos/av_audio.h"
-        "${CMAKE_SOURCE_DIR}/src/platform/macos/av_audio.m"
+        "${CMAKE_SOURCE_DIR}/src/platform/macos/av_audio.mm"
         "${CMAKE_SOURCE_DIR}/src/platform/macos/av_img_t.h"
         "${CMAKE_SOURCE_DIR}/src/platform/macos/av_video.h"
         "${CMAKE_SOURCE_DIR}/src/platform/macos/av_video.m"

cmake/compile_definitions/unix.cmake

@@ -5,6 +5,7 @@ list(APPEND SUNSHINE_EXTERNAL_LIBRARIES
         ${CURL_LIBRARIES})
 
 # add install prefix to assets path if not already there
-if(NOT SUNSHINE_ASSETS_DIR MATCHES "^${CMAKE_INSTALL_PREFIX}")
+# Skip prefix addition for absolute paths or development builds
+if(NOT SUNSHINE_ASSETS_DIR MATCHES "^/" AND NOT SUNSHINE_ASSETS_DIR MATCHES "^${CMAKE_INSTALL_PREFIX}")
     set(SUNSHINE_ASSETS_DIR "${CMAKE_INSTALL_PREFIX}/${SUNSHINE_ASSETS_DIR}")
 endif()

cmake/dependencies/common.cmake

@@ -18,6 +18,9 @@ add_subdirectory("${CMAKE_SOURCE_DIR}/third-party/libdisplaydevice")
 # common dependencies
 include("${CMAKE_MODULE_PATH}/dependencies/nlohmann_json.cmake")
 find_package(OpenSSL REQUIRED)
+if(OPENSSL_FOUND)
+    include_directories(SYSTEM ${OPENSSL_INCLUDE_DIR})
+endif()
 find_package(PkgConfig REQUIRED)
 find_package(Threads REQUIRED)
 pkg_check_modules(CURL REQUIRED libcurl)

cmake/dependencies/macos.cmake

@@ -11,3 +11,11 @@ FIND_LIBRARY(VIDEO_TOOLBOX_LIBRARY VideoToolbox)
 if(SUNSHINE_ENABLE_TRAY)
     FIND_LIBRARY(COCOA Cocoa REQUIRED)
 endif()
+
+# Audio frameworks required for audio capture/processing
+FIND_LIBRARY(AUDIO_TOOLBOX_LIBRARY AudioToolbox)
+FIND_LIBRARY(AUDIO_UNIT_LIBRARY AudioUnit)
+FIND_LIBRARY(CORE_AUDIO_LIBRARY CoreAudio)
+
+include_directories(/opt/homebrew/opt/opus/include)
+link_directories(/opt/homebrew/opt/opus/lib)

docs/configuration.md

@@ -825,6 +825,31 @@ editing the `conf` file in a text editor. Use the examples as reference.
     </tr>
 </table>
 
+### macos_system_wide_audio_tap
+
+<table>
+    <tr>
+        <td>Description</td>
+        <td colspan="2">
+            @tip{Overrides Audio Sink settings.} 
+            Toggles the creation of a system-wide audio tap that captures outgoing audio from all processes. 
+            This tap can act as an input in a HAL aggregate device, like a virtual microphone. 
+            @note{Requirement: macOS 14.2 or later.}
+            @attention{macOS Privacy Settings: The user must add Terminal or Sunshine to <strong>Privacy & Security &gt; Screen & System Audio Recording &gt; System Audio Recording Only</strong> in System Settings.}
+        </td>
+    </tr>
+    <tr>
+        <td>Default</td>
+        <td colspan="2">disabled</td>
+    </tr>
+    <tr>
+        <td>Example</td>
+        <td colspan="2">@code{}
+            macos_system_wide_audio_tap = disabled
+            @endcode</td>
+    </tr>
+</table>
+
 ### install_steam_audio_drivers
 
 <table>

src/config.cpp

@@ -513,6 +513,7 @@ namespace config {
     {},  // virtual_sink
     true,  // stream audio
     true,  // install_steam_drivers
+    true,  // macos_system_wide_audio_tap
   };
 
   stream_t stream {
@@ -1166,6 +1167,7 @@ namespace config {
     string_f(vars, "virtual_sink", audio.virtual_sink);
     bool_f(vars, "stream_audio", audio.stream);
     bool_f(vars, "install_steam_audio_drivers", audio.install_steam_drivers);
+    bool_f(vars, "macos_system_wide_audio_tap", audio.macos_system_wide_audio_tap);
 
     string_restricted_f(vars, "origin_web_ui_allowed", nvhttp.origin_web_ui_allowed, {"pc"sv, "lan"sv, "wan"sv});
 

src/config.h

@@ -145,10 +145,11 @@ namespace config {
   };
 
   struct audio_t {
-    std::string sink;
-    std::string virtual_sink;
-    bool stream;
-    bool install_steam_drivers;
+    std::string sink;  ///< Audio output device/sink to use for audio capture
+    std::string virtual_sink;  ///< Virtual audio sink for audio routing
+    bool stream;  ///< Enable audio streaming to clients
+    bool install_steam_drivers;  ///< Install Steam audio drivers for enhanced compatibility
+    bool macos_system_wide_audio_tap;  ///< Enable system-wide audio capture on macOS using Core Audio taps (requires macOS 14.2+)
   };
 
   constexpr int ENCRYPTION_MODE_NEVER = 0;  // Never use video encryption, even if the client supports it

src/platform/macos/av_audio.h

@@ -1,29 +1,205 @@
 /**
  * @file src/platform/macos/av_audio.h
- * @brief Declarations for audio capture on macOS.
+ * @brief Declarations for macOS audio capture with dual input paths.
+ *
+ * This header defines the AVAudio class which provides two distinct audio capture methods:
+ * 1. **Microphone capture** - Uses AVFoundation framework to capture from specific microphone devices
+ * 2. **System-wide audio tap** - Uses Core Audio taps to capture all system audio output (macOS 14.2+)
+ *
+ * The system-wide audio tap allows capturing audio from all applications and system sounds,
+ * while microphone capture focuses on input from physical or virtual microphone devices.
  */
 #pragma once
 
 // platform includes
+#import <AudioToolbox/AudioToolbox.h>
 #import <AVFoundation/AVFoundation.h>
+#import <CoreAudio/AudioHardwareTapping.h>
+#import <CoreAudio/CoreAudio.h>
 
 // lib includes
 #include "third-party/TPCircularBuffer/TPCircularBuffer.h"
 
+// Buffer length for audio processing
 #define kBufferLength 4096
 
+NS_ASSUME_NONNULL_BEGIN
+
+// Forward declarations
+@class AVAudio;
+@class CATapDescription;
+
+/**
+ * @brief Data structure for AudioConverter input callback.
+ * Contains audio data and metadata needed for format conversion during audio processing.
+ */
+struct AudioConverterInputData {
+  float *inputData;  ///< Pointer to input audio data
+  UInt32 inputFrames;  ///< Total number of input frames available
+  UInt32 framesProvided;  ///< Number of frames already provided to converter
+  UInt32 deviceChannels;  ///< Number of channels in the device audio
+  AVAudio *avAudio;  ///< Reference to the AVAudio instance
+};
+
+/**
+ * @brief IOProc client data structure for Core Audio system taps.
+ * Contains configuration and conversion data for real-time audio processing.
+ */
+typedef struct {
+  AVAudio *avAudio;  ///< Reference to AVAudio instance
+  UInt32 clientRequestedChannels;  ///< Number of channels requested by client
+  UInt32 clientRequestedSampleRate;  ///< Sample rate requested by client
+  UInt32 clientRequestedFrameSize;  ///< Frame size requested by client
+  UInt32 aggregateDeviceSampleRate;  ///< Sample rate of the aggregate device
+  UInt32 aggregateDeviceChannels;  ///< Number of channels in aggregate device
+  AudioConverterRef _Nullable audioConverter;  ///< Audio converter for format conversion
+} AVAudioIOProcData;
+
+/**
+ * @brief Core Audio capture class for macOS audio input and system-wide audio tapping.
+ * Provides functionality for both microphone capture via AVFoundation and system-wide
+ * audio capture via Core Audio taps (requires macOS 14.2+).
+ */
 @interface AVAudio: NSObject <AVCaptureAudioDataOutputSampleBufferDelegate> {
 @public
-  TPCircularBuffer audioSampleBuffer;
+  TPCircularBuffer audioSampleBuffer;  ///< Shared circular buffer for both audio capture paths
+@private
+  // System-wide audio tap components (Core Audio)
+  AudioObjectID tapObjectID;  ///< Core Audio tap object identifier for system audio capture
+  AudioObjectID aggregateDeviceID;  ///< Aggregate device ID for system tap audio routing
+  AudioDeviceIOProcID ioProcID;  ///< IOProc identifier for real-time audio processing
+  AVAudioIOProcData *_Nullable ioProcData;  ///< Context data for IOProc callbacks and format conversion
 }
 
-@property (nonatomic, assign) AVCaptureSession *audioCaptureSession;
-@property (nonatomic, assign) AVCaptureConnection *audioConnection;
-@property (nonatomic, assign) NSCondition *samplesArrivedSignal;
+// AVFoundation microphone capture properties
+@property (nonatomic, assign, nullable) AVCaptureSession *audioCaptureSession;  ///< AVFoundation capture session for microphone input
+@property (nonatomic, assign, nullable) AVCaptureConnection *audioConnection;  ///< Audio connection within the capture session
 
-+ (NSArray *)microphoneNames;
-+ (AVCaptureDevice *)findMicrophone:(NSString *)name;
+// Shared synchronization property (used by both audio paths)
+@property (nonatomic, assign, nullable) NSCondition *samplesArrivedSignal;  ///< Condition variable to signal when audio samples are available
 
-- (int)setupMicrophone:(AVCaptureDevice *)device sampleRate:(UInt32)sampleRate frameSize:(UInt32)frameSize channels:(UInt8)channels;
+/**
+ * @brief Get all available microphone devices on the system.
+ * @return Array of AVCaptureDevice objects representing available microphones
+ */
++ (NSArray<AVCaptureDevice *> *)microphones;
+
+/**
+ * @brief Get names of all available microphone devices.
+ * @return Array of NSString objects with microphone device names
+ */
++ (NSArray<NSString *> *)microphoneNames;
+
+/**
+ * @brief Find a specific microphone device by name.
+ * @param name The name of the microphone to find (nullable - will return nil if name is nil)
+ * @return AVCaptureDevice object if found, nil otherwise
+ */
++ (nullable AVCaptureDevice *)findMicrophone:(nullable NSString *)name;
+
+/**
+ * @brief Sets up microphone capture using AVFoundation framework.
+ * @param device The AVCaptureDevice to use for audio input (nullable - will return error if nil)
+ * @param sampleRate Target sample rate in Hz
+ * @param frameSize Number of frames per buffer
+ * @param channels Number of audio channels (1=mono, 2=stereo)
+ * @return 0 on success, -1 on failure
+ */
+- (int)setupMicrophone:(nullable AVCaptureDevice *)device sampleRate:(UInt32)sampleRate frameSize:(UInt32)frameSize channels:(UInt8)channels;
+
+/**
+ * @brief Sets up system-wide audio tap for capturing all system audio.
+ * Requires macOS 14.2+ and appropriate permissions.
+ * @param sampleRate Target sample rate in Hz
+ * @param frameSize Number of frames per buffer
+ * @param channels Number of audio channels
+ * @return 0 on success, -1 on failure
+ */
+- (int)setupSystemTap:(UInt32)sampleRate frameSize:(UInt32)frameSize channels:(UInt8)channels;
+
+// Buffer management methods for testing and internal use
+/**
+ * @brief Initializes the circular audio buffer for the specified number of channels.
+ * @param channels Number of audio channels to configure the buffer for
+ */
+- (void)initializeAudioBuffer:(UInt8)channels;
+
+/**
+ * @brief Cleans up and deallocates the audio buffer resources.
+ */
+- (void)cleanupAudioBuffer;
+
+/**
+ * @brief Cleans up system tap resources in a safe, ordered manner.
+ * @param tapDescription Optional tap description object to release (can be nil)
+ */
+- (void)cleanupSystemTapContext:(nullable id)tapDescription;
+
+/**
+ * @brief Initializes the system tap context with specified audio parameters.
+ * @param sampleRate Target sample rate in Hz
+ * @param frameSize Number of frames per buffer
+ * @param channels Number of audio channels
+ * @return 0 on success, -1 on failure
+ */
+- (int)initializeSystemTapContext:(UInt32)sampleRate frameSize:(UInt32)frameSize channels:(UInt8)channels;
+
+/**
+ * @brief Creates a Core Audio tap description for system audio capture.
+ * @param channels Number of audio channels to configure the tap for
+ * @return CATapDescription object on success, nil on failure
+ */
+- (nullable CATapDescription *)createSystemTapDescriptionForChannels:(UInt8)channels;
+
+/**
+ * @brief Creates an aggregate device with the specified tap description and audio parameters.
+ * @param tapDescription Core Audio tap description for system audio capture
+ * @param sampleRate Target sample rate in Hz
+ * @param frameSize Number of frames per buffer
+ * @return OSStatus indicating success (noErr) or error code
+ */
+- (OSStatus)createAggregateDeviceWithTapDescription:(CATapDescription *)tapDescription sampleRate:(UInt32)sampleRate frameSize:(UInt32)frameSize;
+
+/**
+ * @brief Audio converter complex input callback for format conversion.
+ * Handles audio data conversion between different formats during system audio capture.
+ * @param inAudioConverter The audio converter reference
+ * @param ioNumberDataPackets Number of data packets to convert
+ * @param ioData Audio buffer list for converted data
+ * @param outDataPacketDescription Packet description for output data
+ * @param inputInfo Input data structure containing source audio
+ * @return OSStatus indicating success (noErr) or error code
+ */
+- (OSStatus)audioConverterComplexInputProc:(AudioConverterRef)inAudioConverter
+                       ioNumberDataPackets:(UInt32 *)ioNumberDataPackets
+                                    ioData:(AudioBufferList *)ioData
+                  outDataPacketDescription:(AudioStreamPacketDescription *_Nullable *_Nullable)outDataPacketDescription
+                                 inputInfo:(struct AudioConverterInputData *)inputInfo;
+
+/**
+ * @brief Core Audio IOProc callback for processing system audio data.
+ * Handles real-time audio processing, format conversion, and writes to circular buffer.
+ * @param inDevice The audio device identifier
+ * @param inNow Current audio time stamp
+ * @param inInputData Input audio buffer list from the device
+ * @param inInputTime Time stamp for input data
+ * @param outOutputData Output audio buffer list (nullable for input-only devices)
+ * @param inOutputTime Time stamp for output data
+ * @param clientChannels Number of channels requested by client
+ * @param clientFrameSize Frame size requested by client
+ * @param clientSampleRate Sample rate requested by client
+ * @return OSStatus indicating success (noErr) or error code
+ */
+- (OSStatus)systemAudioIOProc:(AudioObjectID)inDevice
+                        inNow:(const AudioTimeStamp *)inNow
+                  inInputData:(const AudioBufferList *)inInputData
+                  inInputTime:(const AudioTimeStamp *)inInputTime
+                outOutputData:(nullable AudioBufferList *)outOutputData
+                 inOutputTime:(const AudioTimeStamp *)inOutputTime
+               clientChannels:(UInt32)clientChannels
+              clientFrameSize:(UInt32)clientFrameSize
+             clientSampleRate:(UInt32)clientSampleRate;
 
 @end
+
+NS_ASSUME_NONNULL_END