feat(macOS): Capture audio on macOS using Tap API

.gitignore

@@ -56,3 +56,6 @@ package-lock.json
 # Python
 *.pyc
 venv/
+
+# Caches
+.cache/

cmake/compile_definitions/macos.cmake

@@ -24,13 +24,16 @@ list(APPEND SUNSHINE_EXTERNAL_LIBRARIES
         ${CORE_MEDIA_LIBRARY}
         ${CORE_VIDEO_LIBRARY}
         ${FOUNDATION_LIBRARY}
+        ${AUDIO_TOOLBOX_LIBRARY}
+        ${AUDIO_UNIT_LIBRARY}
+        ${CORE_AUDIO_LIBRARY}
         ${VIDEO_TOOLBOX_LIBRARY})
 
 set(APPLE_PLIST_FILE "${SUNSHINE_SOURCE_ASSETS_DIR}/macos/assets/Info.plist")
 
 set(PLATFORM_TARGET_FILES
         "${CMAKE_SOURCE_DIR}/src/platform/macos/av_audio.h"
-        "${CMAKE_SOURCE_DIR}/src/platform/macos/av_audio.m"
+        "${CMAKE_SOURCE_DIR}/src/platform/macos/av_audio.mm"
         "${CMAKE_SOURCE_DIR}/src/platform/macos/av_img_t.h"
         "${CMAKE_SOURCE_DIR}/src/platform/macos/av_video.h"
         "${CMAKE_SOURCE_DIR}/src/platform/macos/av_video.m"

cmake/compile_definitions/unix.cmake

@@ -5,6 +5,7 @@ list(APPEND SUNSHINE_EXTERNAL_LIBRARIES
         ${CURL_LIBRARIES})
 
 # add install prefix to assets path if not already there
-if(NOT SUNSHINE_ASSETS_DIR MATCHES "^${CMAKE_INSTALL_PREFIX}")
+# Skip prefix addition for absolute paths or development builds
+if(NOT SUNSHINE_ASSETS_DIR MATCHES "^/" AND NOT SUNSHINE_ASSETS_DIR MATCHES "^${CMAKE_INSTALL_PREFIX}")
     set(SUNSHINE_ASSETS_DIR "${CMAKE_INSTALL_PREFIX}/${SUNSHINE_ASSETS_DIR}")
 endif()

cmake/dependencies/common.cmake

@@ -18,6 +18,9 @@ add_subdirectory("${CMAKE_SOURCE_DIR}/third-party/libdisplaydevice")
 # common dependencies
 include("${CMAKE_MODULE_PATH}/dependencies/nlohmann_json.cmake")
 find_package(OpenSSL REQUIRED)
+if(OPENSSL_FOUND)
+    include_directories(SYSTEM ${OPENSSL_INCLUDE_DIR})
+endif()
 find_package(PkgConfig REQUIRED)
 find_package(Threads REQUIRED)
 pkg_check_modules(CURL REQUIRED libcurl)

cmake/dependencies/macos.cmake

@@ -11,3 +11,11 @@ FIND_LIBRARY(VIDEO_TOOLBOX_LIBRARY VideoToolbox)
 if(SUNSHINE_ENABLE_TRAY)
     FIND_LIBRARY(COCOA Cocoa REQUIRED)
 endif()
+
+# Audio frameworks required for audio capture/processing
+FIND_LIBRARY(AUDIO_TOOLBOX_LIBRARY AudioToolbox)
+FIND_LIBRARY(AUDIO_UNIT_LIBRARY AudioUnit)
+FIND_LIBRARY(CORE_AUDIO_LIBRARY CoreAudio)
+
+include_directories(/opt/homebrew/opt/opus/include)
+link_directories(/opt/homebrew/opt/opus/lib)

docs/getting_started.md

@@ -418,9 +418,10 @@ systemctl --user enable sunshine
 ### macOS
 The first time you start Sunshine, you will be asked to grant access to screen recording and your microphone.
 
-Sunshine can only access microphones on macOS due to system limitations. To stream system audio use
-[Soundflower](https://github.com/mattingalls/Soundflower) or
-[BlackHole](https://github.com/ExistentialAudio/BlackHole).
+Sunshine supports native system audio capture on macOS 14.0 (Sonoma) and newer via Apple’s Audio Tap API.
+To use it, simply leave the **Audio Sink** setting blank.
+
+If you are running macOS 13 (Ventura) or earlier—or if you prefer to manage your own loopback device—you can still use [Soundflower](https://github.com/mattingalls/Soundflower) or [BlackHole](https://github.com/ExistentialAudio/BlackHole) and enter its device name in the **Audio Sink** field.
 
 > [!NOTE]
 > Command Keys are not forwarded by Moonlight. Right Option-Key is mapped to CMD-Key.

packaging/sunshine.rb

@@ -147,8 +147,11 @@ def post_install
 
     if OS.mac?
       opoo <<~EOS
-        Sunshine can only access microphones on macOS due to system limitations.
-        To stream system audio use "Soundflower" or "BlackHole".
+        Sunshine now supports system audio capture natively on macOS 14.0 (Sonoma) and later,
+        using the built-in Core Audio Tap API.
+
+        On macOS 13 or earlier, or if you prefer a virtual loopback device,
+        you can still use "Soundflower" or "BlackHole" for system audio capture.
 
         Gamepads are not currently supported on macOS.
       EOS

src/audio.cpp

@@ -193,8 +193,9 @@ namespace audio {
     }
 
     auto frame_size = config.packetDuration * stream.sampleRate / 1000;
+    bool host_audio = config.flags[config_t::HOST_AUDIO];
     bool continuous_audio = config.flags[config_t::CONTINUOUS_AUDIO];
-    auto mic = control->microphone(stream.mapping, stream.channelCount, stream.sampleRate, frame_size, continuous_audio);
+    auto mic = control->microphone(stream.mapping, stream.channelCount, stream.sampleRate, frame_size, continuous_audio, host_audio);
     if (!mic) {
       return;
     }
@@ -231,7 +232,7 @@ namespace audio {
           BOOST_LOG(info) << "Reinitializing audio capture"sv;
           mic.reset();
           do {
-            mic = control->microphone(stream.mapping, stream.channelCount, stream.sampleRate, frame_size, continuous_audio);
+            mic = control->microphone(stream.mapping, stream.channelCount, stream.sampleRate, frame_size, continuous_audio, host_audio);
             if (!mic) {
               BOOST_LOG(warning) << "Couldn't re-initialize audio input"sv;
             }

src/config.h

@@ -145,10 +145,10 @@ namespace config {
   };
 
   struct audio_t {
-    std::string sink;
-    std::string virtual_sink;
-    bool stream;
-    bool install_steam_drivers;
+    std::string sink;  ///< Audio output device/sink to use for audio capture
+    std::string virtual_sink;  ///< Virtual audio sink for audio routing
+    bool stream;  ///< Enable audio streaming to clients
+    bool install_steam_drivers;  ///< Install Steam audio drivers for enhanced compatibility
   };
 
   constexpr int ENCRYPTION_MODE_NEVER = 0;  // Never use video encryption, even if the client supports it

src/platform/common.h

@@ -554,7 +554,7 @@ namespace platf {
   public:
     virtual int set_sink(const std::string &sink) = 0;
 
-    virtual std::unique_ptr<mic_t> microphone(const std::uint8_t *mapping, int channels, std::uint32_t sample_rate, std::uint32_t frame_size, bool continuous) = 0;
+    virtual std::unique_ptr<mic_t> microphone(const std::uint8_t *mapping, int channels, std::uint32_t sample_rate, std::uint32_t frame_size, bool continuous, [[maybe_unused]] bool host_audio_enabled) = 0;
 
     /**
      * @brief Check if the audio sink is available in the system.

src/platform/linux/audio.cpp

@@ -440,7 +440,7 @@ namespace platf {
         return monitor_name;
       }
 
-      std::unique_ptr<mic_t> microphone(const std::uint8_t *mapping, int channels, std::uint32_t sample_rate, std::uint32_t frame_size, bool continuous_audio) override {
+      std::unique_ptr<mic_t> microphone(const std::uint8_t *mapping, int channels, std::uint32_t sample_rate, std::uint32_t frame_size, bool continuous_audio, [[maybe_unused]] bool host_audio_enabled) override {
         // Sink choice priority:
         // 1. Config sink
         // 2. Last sink swapped to (Usually virtual in this case)

src/platform/macos/av_audio.h

@@ -1,29 +1,171 @@
 /**
  * @file src/platform/macos/av_audio.h
- * @brief Declarations for audio capture on macOS.
+ * @brief Declarations for macOS audio capture with dual input paths.
+ *
+ * This header defines the AVAudio class which provides distinct audio capture methods:
+ * 1. **Microphone capture** - Uses AVFoundation framework to capture from specific microphone devices
+ * 2. **System-wide audio tap** - Uses Core Audio taps to capture all system audio output (macOS 14.0+)
+ *
+ * The system-wide audio tap allows capturing audio from all applications and system sounds,
+ * while microphone capture focuses on input from physical or virtual microphone devices.
  */
 #pragma once
 
 // platform includes
+#import <AudioToolbox/AudioToolbox.h>
 #import <AVFoundation/AVFoundation.h>
+#import <CoreAudio/AudioHardwareTapping.h>
+#import <CoreAudio/CoreAudio.h>
 
 // lib includes
 #include "third-party/TPCircularBuffer/TPCircularBuffer.h"
 
+// Buffer length for audio processing
 #define kBufferLength 4096
 
+NS_ASSUME_NONNULL_BEGIN
+
+// Forward declarations
+@class AVAudio;
+@class CATapDescription;
+
+namespace platf {
+  OSStatus audioConverterComplexInputProc(AudioConverterRef _Nullable inAudioConverter, UInt32 *_Nonnull ioNumberDataPackets, AudioBufferList *_Nonnull ioData, AudioStreamPacketDescription *_Nullable *_Nullable outDataPacketDescription, void *_Nonnull inUserData);
+  OSStatus systemAudioIOProc(AudioObjectID inDevice, const AudioTimeStamp *_Nullable inNow, const AudioBufferList *_Nullable inInputData, const AudioTimeStamp *_Nullable inInputTime, AudioBufferList *_Nullable outOutputData, const AudioTimeStamp *_Nullable inOutputTime, void *_Nullable inClientData);
+}  // namespace platf
+
+/**
+ * @brief Data structure for AudioConverter input callback.
+ * Contains audio data and metadata needed for format conversion during audio processing.
+ */
+struct AudioConverterInputData {
+  float *inputData;  ///< Pointer to input audio data
+  UInt32 inputFrames;  ///< Total number of input frames available
+  UInt32 framesProvided;  ///< Number of frames already provided to converter
+  UInt32 deviceChannels;  ///< Number of channels in the device audio
+  AVAudio *avAudio;  ///< Reference to the AVAudio instance
+};
+
+/**
+ * @brief IOProc client data structure for Core Audio system taps.
+ * Contains configuration and conversion data for real-time audio processing.
+ */
+typedef struct {
+  AVAudio *avAudio;  ///< Reference to AVAudio instance
+  UInt32 clientRequestedChannels;  ///< Number of channels requested by client
+  UInt32 clientRequestedSampleRate;  ///< Sample rate requested by client
+  UInt32 clientRequestedFrameSize;  ///< Frame size requested by client
+  UInt32 aggregateDeviceSampleRate;  ///< Sample rate of the aggregate device
+  UInt32 aggregateDeviceChannels;  ///< Number of channels in aggregate device
+  AudioConverterRef _Nullable audioConverter;  ///< Audio converter for format conversion
+  float *_Nullable conversionBuffer;  ///< Pre-allocated buffer for audio conversion
+  UInt32 conversionBufferSize;  ///< Size of the conversion buffer in bytes
+} AVAudioIOProcData;
+
+/**
+ * @brief Core Audio capture class for macOS audio input and system-wide audio tapping.
+ * Provides functionality for both microphone capture via AVFoundation and system-wide
+ * audio capture via Core Audio taps (requires macOS 14.0+).
+ */
 @interface AVAudio: NSObject <AVCaptureAudioDataOutputSampleBufferDelegate> {
 @public
-  TPCircularBuffer audioSampleBuffer;
+  TPCircularBuffer audioSampleBuffer;  ///< Shared circular buffer for both audio capture paths
+  dispatch_semaphore_t audioSemaphore;  ///< Real-time safe semaphore for signaling audio sample availability
+@private
+  // System-wide audio tap components (Core Audio)
+  AudioObjectID tapObjectID;  ///< Core Audio tap object identifier for system audio capture
+  AudioObjectID aggregateDeviceID;  ///< Aggregate device ID for system tap audio routing
+  AudioDeviceIOProcID ioProcID;  ///< IOProc identifier for real-time audio processing
+  AVAudioIOProcData *_Nullable ioProcData;  ///< Context data for IOProc callbacks and format conversion
 }
 
-@property (nonatomic, assign) AVCaptureSession *audioCaptureSession;
-@property (nonatomic, assign) AVCaptureConnection *audioConnection;
-@property (nonatomic, assign) NSCondition *samplesArrivedSignal;
+// AVFoundation microphone capture properties
+@property (nonatomic, assign, nullable) AVCaptureSession *audioCaptureSession;  ///< AVFoundation capture session for microphone input
+@property (nonatomic, assign, nullable) AVCaptureConnection *audioConnection;  ///< Audio connection within the capture session
+@property (nonatomic, assign) BOOL hostAudioEnabled;  ///< Whether host audio playback should be enabled (affects tap mute behavior)
 
-+ (NSArray *)microphoneNames;
-+ (AVCaptureDevice *)findMicrophone:(NSString *)name;
+/**
+ * @brief Get all available microphone devices on the system.
+ * @return Array of AVCaptureDevice objects representing available microphones
+ */
++ (NSArray<AVCaptureDevice *> *)microphones;
 
-- (int)setupMicrophone:(AVCaptureDevice *)device sampleRate:(UInt32)sampleRate frameSize:(UInt32)frameSize channels:(UInt8)channels;
+/**
+ * @brief Get names of all available microphone devices.
+ * @return Array of NSString objects with microphone device names
+ */
++ (NSArray<NSString *> *)microphoneNames;
+
+/**
+ * @brief Find a specific microphone device by name.
+ * @param name The name of the microphone to find (nullable - will return nil if name is nil)
+ * @return AVCaptureDevice object if found, nil otherwise
+ */
++ (nullable AVCaptureDevice *)findMicrophone:(nullable NSString *)name;
+
+/**
+ * @brief Sets up microphone capture using AVFoundation framework.
+ * @param device The AVCaptureDevice to use for audio input (nullable - will return error if nil)
+ * @param sampleRate Target sample rate in Hz
+ * @param frameSize Number of frames per buffer
+ * @param channels Number of audio channels (1=mono, 2=stereo)
+ * @return 0 on success, -1 on failure
+ */
+- (int)setupMicrophone:(nullable AVCaptureDevice *)device sampleRate:(UInt32)sampleRate frameSize:(UInt32)frameSize channels:(UInt8)channels;
+
+/**
+ * @brief Sets up system-wide audio tap for capturing all system audio.
+ * Requires macOS 14.0+ and appropriate permissions.
+ * @param sampleRate Target sample rate in Hz
+ * @param frameSize Number of frames per buffer
+ * @param channels Number of audio channels
+ * @return 0 on success, -1 on failure
+ */
+- (int)setupSystemTap:(UInt32)sampleRate frameSize:(UInt32)frameSize channels:(UInt8)channels;
+
+// Buffer management methods for testing and internal use
+/**
+ * @brief Initializes the circular audio buffer for the specified number of channels.
+ * @param channels Number of audio channels to configure the buffer for
+ */
+- (void)initializeAudioBuffer:(UInt8)channels;
+
+/**
+ * @brief Cleans up and deallocates the audio buffer resources.
+ */
+- (void)cleanupAudioBuffer;
+
+/**
+ * @brief Cleans up system tap resources in a safe, ordered manner.
+ * @param tapDescription Optional tap description object to release (can be nil)
+ */
+- (void)cleanupSystemTapContext:(nullable id)tapDescription;
+
+/**
+ * @brief Initializes the system tap context with specified audio parameters.
+ * @param sampleRate Target sample rate in Hz
+ * @param frameSize Number of frames per buffer
+ * @param channels Number of audio channels
+ * @return 0 on success, -1 on failure
+ */
+- (int)initializeSystemTapContext:(UInt32)sampleRate frameSize:(UInt32)frameSize channels:(UInt8)channels;
+
+/**
+ * @brief Creates a Core Audio tap description for system audio capture.
+ * @param channels Number of audio channels to configure the tap for
+ * @return CATapDescription object on success, nil on failure
+ */
+- (nullable CATapDescription *)createSystemTapDescriptionForChannels:(UInt8)channels;
+
+/**
+ * @brief Creates an aggregate device with the specified tap description and audio parameters.
+ * @param tapDescription Core Audio tap description for system audio capture
+ * @param sampleRate Target sample rate in Hz
+ * @param frameSize Number of frames per buffer
+ * @return OSStatus indicating success (noErr) or error code
+ */
+- (OSStatus)createAggregateDeviceWithTapDescription:(CATapDescription *)tapDescription sampleRate:(UInt32)sampleRate frameSize:(UInt32)frameSize;
 
 @end
+
+NS_ASSUME_NONNULL_END