Expose audio sample buffers for Android (#89)

* Initial draft

* Working impl

* doc and cleanup

* doc update

Author: davidliu

sdk/android/BUILD.gn

@@ -255,6 +255,7 @@ if (is_android) {
       "api/org/webrtc/AudioProcessingFactory.java",
       "api/org/webrtc/AudioSource.java",
       "api/org/webrtc/AudioTrack.java",
+      "api/org/webrtc/AudioTrackSink.java",
       "api/org/webrtc/CallSessionFileRotatingLogSink.java",
       "api/org/webrtc/CandidatePairChangeEvent.java",
       "api/org/webrtc/CryptoOptions.java",
@@ -716,6 +717,8 @@ if (current_os == "linux" || is_android) {
       "src/jni/pc/add_ice_candidate_observer.cc",
       "src/jni/pc/add_ice_candidate_observer.h",
       "src/jni/pc/android_network_monitor.h",
+      "src/jni/pc/audio_sink.cc",
+      "src/jni/pc/audio_sink.h",
       "src/jni/pc/audio_track.cc",
       "src/jni/pc/call_session_file_rotating_log_sink.cc",
       "src/jni/pc/crypto_options.cc",
@@ -1405,6 +1408,7 @@ if (current_os == "linux" || is_android) {
     sources = [
       "api/org/webrtc/AddIceObserver.java",
       "api/org/webrtc/AudioTrack.java",
+      "api/org/webrtc/AudioTrackSink.java",
       "api/org/webrtc/CallSessionFileRotatingLogSink.java",
       "api/org/webrtc/CandidatePairChangeEvent.java",
       "api/org/webrtc/CryptoOptions.java",

sdk/android/api/org/webrtc/AudioTrack.java

@@ -10,8 +10,12 @@
 
 package org.webrtc;
 
+import java.util.IdentityHashMap;
+
 /** Java wrapper for a C++ AudioTrackInterface */
 public class AudioTrack extends MediaStreamTrack {
+  private final IdentityHashMap<AudioTrackSink, Long> sinks = new IdentityHashMap<AudioTrackSink, Long>();
+
   public AudioTrack(long nativeTrack) {
     super(nativeTrack);
   }
@@ -23,10 +27,54 @@ public void setVolume(double volume) {
     nativeSetVolume(getNativeAudioTrack(), volume);
   }
 
+  /**
+   * Adds an AudioTrackSink to the track. This callback is only
+   * called for remote audio tracks.
+   * 
+   * Repeated addSink calls will not add the sink multiple times.
+   */
+  public void addSink(AudioTrackSink sink) {
+    if (sink == null) {
+      throw new IllegalArgumentException("The AudioTrackSink is not allowed to be null");
+    }
+    if (!sinks.containsKey(sink)) {
+      final long nativeSink = nativeWrapSink(sink);
+      sinks.put(sink, nativeSink);
+      nativeAddSink(getNativeMediaStreamTrack(), nativeSink);
+    }
+  }
+
+  /**
+   * Removes an AudioTrackSink from the track.
+   *
+   * If the AudioTrackSink was not attached to the track, this is a no-op.
+   */
+  public void removeSink(AudioTrackSink sink) {
+    final Long nativeSink = sinks.remove(sink);
+    if (nativeSink != null) {
+      nativeRemoveSink(getNativeMediaStreamTrack(), nativeSink);
+      nativeFreeSink(nativeSink);
+    }
+  }
+
+  @Override
+  public void dispose() {
+    for (long nativeSink : sinks.values()) {
+      nativeRemoveSink(getNativeMediaStreamTrack(), nativeSink);
+      nativeFreeSink(nativeSink);
+    }
+    sinks.clear();
+    super.dispose();
+  }
+
   /** Returns a pointer to webrtc::AudioTrackInterface. */
   long getNativeAudioTrack() {
     return getNativeMediaStreamTrack();
   }
 
   private static native void nativeSetVolume(long track, double volume);
+  private static native void nativeAddSink(long track, long nativeSink);
+  private static native void nativeRemoveSink(long track, long nativeSink);
+  private static native long nativeWrapSink(AudioTrackSink sink);
+  private static native void nativeFreeSink(long sink);
 }

sdk/android/api/org/webrtc/AudioTrackSink.java

@@ -0,0 +1,27 @@
+/*
+ *  Copyright 2023 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+package org.webrtc;
+
+import java.nio.ByteBuffer;
+
+/**
+ * Java version of rtc::AudioTrackSinkInterface.
+ */
+public interface AudioTrackSink {
+  /**
+   * Implementations should copy the audio data into a local copy if they wish
+   * to use the data after this function returns.
+   */
+  @CalledByNative 
+  void onData(ByteBuffer audioData, int bitsPerSample, int sampleRate,
+      int numberOfChannels, int numberOfFrames, 
+      long absoluteCaptureTimestampMs);
+}

sdk/android/api/org/webrtc/audio/JavaAudioDeviceModule.java

@@ -42,6 +42,7 @@ public static class Builder {
     private AudioTrackErrorCallback audioTrackErrorCallback;
     private AudioRecordErrorCallback audioRecordErrorCallback;
     private SamplesReadyCallback samplesReadyCallback;
+    private PlaybackSamplesReadyCallback playbackSamplesReadyCallback;
     private AudioTrackStateCallback audioTrackStateCallback;
     private AudioRecordStateCallback audioRecordStateCallback;
     private boolean useHardwareAcousticEchoCanceler = isBuiltInAcousticEchoCancelerSupported();
@@ -140,6 +141,14 @@ public Builder setSamplesReadyCallback(SamplesReadyCallback samplesReadyCallback
       return this;
     }
 
+    /**
+     * Set a callback to listen to the audio output passed to the AudioTrack.
+     */
+    public Builder setPlaybackSamplesReadyCallback(PlaybackSamplesReadyCallback playbackSamplesReadyCallback) {
+      this.playbackSamplesReadyCallback = playbackSamplesReadyCallback;
+      return this;
+    }
+
     /**
      * Set a callback to retrieve information from the AudioTrack on when audio starts and stop.
      */
@@ -258,7 +267,7 @@ public JavaAudioDeviceModule createAudioDeviceModule() {
           samplesReadyCallback, useHardwareAcousticEchoCanceler, useHardwareNoiseSuppressor);
       final WebRtcAudioTrack audioOutput =
           new WebRtcAudioTrack(context, audioManager, audioAttributes, audioTrackErrorCallback,
-              audioTrackStateCallback, useLowLatency, enableVolumeLogger);
+              audioTrackStateCallback, playbackSamplesReadyCallback, useLowLatency, enableVolumeLogger);
       return new JavaAudioDeviceModule(context, audioManager, audioInput, audioOutput,
           inputSampleRate, outputSampleRate, useStereoInput, useStereoOutput);
     }
@@ -325,6 +334,11 @@ public static interface SamplesReadyCallback {
     void onWebRtcAudioRecordSamplesReady(AudioSamples samples);
   }
 
+  /** Called when new audio samples are ready. This should only be set for debug purposes */
+  public static interface PlaybackSamplesReadyCallback {
+    void onWebRtcAudioTrackSamplesReady(AudioSamples samples);
+  }
+
   /* AudioTrack */
   // Audio playout/track error handler functions.
   public enum AudioTrackStartErrorCode {

sdk/android/src/java/org/webrtc/audio/WebRtcAudioTrack.java

@@ -20,12 +20,14 @@
 import android.os.Process;
 import androidx.annotation.Nullable;
 import java.nio.ByteBuffer;
+import java.util.Arrays;
 import org.webrtc.CalledByNative;
 import org.webrtc.Logging;
 import org.webrtc.ThreadUtils;
 import org.webrtc.audio.JavaAudioDeviceModule.AudioTrackErrorCallback;
 import org.webrtc.audio.JavaAudioDeviceModule.AudioTrackStartErrorCode;
 import org.webrtc.audio.JavaAudioDeviceModule.AudioTrackStateCallback;
+import org.webrtc.audio.JavaAudioDeviceModule.PlaybackSamplesReadyCallback;
 import org.webrtc.audio.LowLatencyAudioBufferManager;
 
 class WebRtcAudioTrack {
@@ -76,6 +78,7 @@ class WebRtcAudioTrack {
 
   private final @Nullable AudioTrackErrorCallback errorCallback;
   private final @Nullable AudioTrackStateCallback stateCallback;
+  private final @Nullable PlaybackSamplesReadyCallback audioSamplesReadyCallback;
 
   /**
    * Audio thread which keeps calling AudioTrack.write() to stream audio.
@@ -129,6 +132,17 @@ public void run() {
             reportWebRtcAudioTrackError("AudioTrack.write failed: " + bytesWritten);
           }
         }
+
+        if (audioSamplesReadyCallback != null && keepAlive) {
+          // Copy the entire byte buffer array. The start of the byteBuffer is not necessarily
+          // at index 0.
+          byte[] data = Arrays.copyOfRange(byteBuffer.array(), byteBuffer.arrayOffset(),
+              sizeInBytes + byteBuffer.arrayOffset());
+          audioSamplesReadyCallback.onWebRtcAudioTrackSamplesReady(
+              new JavaAudioDeviceModule.AudioSamples(audioTrack.getAudioFormat(),
+                  audioTrack.getChannelCount(), audioTrack.getSampleRate(), data));
+        }
+
         if (useLowLatency) {
           bufferManager.maybeAdjustBufferSize(audioTrack);
         }
@@ -154,20 +168,21 @@ public void stopThread() {
   @CalledByNative
   WebRtcAudioTrack(Context context, AudioManager audioManager) {
     this(context, audioManager, null /* audioAttributes */, null /* errorCallback */,
-        null /* stateCallback */, false /* useLowLatency */, true /* enableVolumeLogger */);
+        null /* stateCallback */, null /* audioSamplesReadyCallback */, false /* useLowLatency */, true /* enableVolumeLogger */);
   }
 
   WebRtcAudioTrack(Context context, AudioManager audioManager,
       @Nullable AudioAttributes audioAttributes, @Nullable AudioTrackErrorCallback errorCallback,
-      @Nullable AudioTrackStateCallback stateCallback, boolean useLowLatency,
-      boolean enableVolumeLogger) {
+      @Nullable AudioTrackStateCallback stateCallback, @Nullable PlaybackSamplesReadyCallback audioSamplesReadyCallback, 
+      boolean useLowLatency, boolean enableVolumeLogger) {
     threadChecker.detachThread();
     this.context = context;
     this.audioManager = audioManager;
     this.audioAttributes = audioAttributes;
     this.errorCallback = errorCallback;
     this.stateCallback = stateCallback;
     this.volumeLogger = enableVolumeLogger ? new VolumeLogger(audioManager) : null;
+    this.audioSamplesReadyCallback = audioSamplesReadyCallback;
     this.useLowLatency = useLowLatency;
     Logging.d(TAG, "ctor" + WebRtcAudioUtils.getThreadInfo());
   }

sdk/android/src/jni/pc/audio_sink.cc

@@ -0,0 +1,39 @@
+/*
+ *  Copyright 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "sdk/android/src/jni/pc/audio_sink.h"
+
+#include "sdk/android/generated_peerconnection_jni/AudioTrackSink_jni.h"
+
+namespace webrtc {
+namespace jni {
+
+AudioTrackSinkWrapper::AudioTrackSinkWrapper(JNIEnv* jni, const JavaRef<jobject>& j_sink)
+    : j_sink_(jni, j_sink) {}
+
+AudioTrackSinkWrapper::~AudioTrackSinkWrapper() {}
+
+void AudioTrackSinkWrapper::OnData(
+    const void* audio_data,
+    int bits_per_sample,
+    int sample_rate,
+    size_t number_of_channels,
+    size_t number_of_frames,
+    absl::optional<int64_t> absolute_capture_timestamp_ms) {
+  JNIEnv* jni = AttachCurrentThreadIfNeeded();
+  int length = (bits_per_sample / 8) * number_of_channels * number_of_frames;
+  ScopedJavaLocalRef<jobject> audio_buffer =
+      NewDirectByteBuffer(jni, (void *) audio_data, length);
+  Java_AudioTrackSink_onData(jni, j_sink_,
+      audio_buffer, bits_per_sample, sample_rate, (int) number_of_channels, (int) number_of_frames, (absolute_capture_timestamp_ms ? absolute_capture_timestamp_ms.value() : 0));
+}
+
+}  // namespace jni
+}  // namespace webrtc

sdk/android/src/jni/pc/audio_sink.h

@@ -0,0 +1,41 @@
+/*
+ *  Copyright 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef SDK_ANDROID_SRC_JNI_AUDIO_TRACK_SINK_H_
+#define SDK_ANDROID_SRC_JNI_AUDIO_TRACK_SINK_H_
+
+#include <jni.h>
+
+#include "api/media_stream_interface.h"
+#include "sdk/android/src/jni/jni_helpers.h"
+
+namespace webrtc {
+namespace jni {
+
+class AudioTrackSinkWrapper : public webrtc::AudioTrackSinkInterface {
+ public:
+  AudioTrackSinkWrapper(JNIEnv* jni, const JavaRef<jobject>& j_sink);
+  ~AudioTrackSinkWrapper() override;
+
+ private:
+  void OnData(const void* audio_data,
+              int bits_per_sample,
+              int sample_rate,
+              size_t number_of_channels,
+              size_t number_of_frames,
+              absl::optional<int64_t> absolute_capture_timestamp_ms) override;
+
+  const ScopedJavaGlobalRef<jobject> j_sink_;
+};
+
+}  // namespace jni
+}  // namespace webrtc
+
+#endif  // SDK_ANDROID_SRC_JNI_AUDIO_TRACK_SINK_H_

sdk/android/src/jni/pc/audio_track.cc

@@ -9,6 +9,8 @@
  */
 
 #include "api/media_stream_interface.h"
+#include "sdk/android/src/jni/pc/audio_sink.h"
+
 #include "sdk/android/generated_peerconnection_jni/AudioTrack_jni.h"
 
 namespace webrtc {
@@ -20,5 +22,29 @@ static void JNI_AudioTrack_SetVolume(JNIEnv*, jlong j_p, jdouble volume) {
   source->SetVolume(volume);
 }
 
+static void JNI_AudioTrack_AddSink(JNIEnv* jni,
+                                   jlong j_native_track,
+                                   jlong j_native_sink) {
+  reinterpret_cast<AudioTrackInterface*>(j_native_track)
+      ->AddSink(reinterpret_cast<webrtc::AudioTrackSinkInterface*>(j_native_sink));
+}
+
+static void JNI_AudioTrack_RemoveSink(JNIEnv* jni,
+                                      jlong j_native_track,
+                                      jlong j_native_sink) {
+  reinterpret_cast<AudioTrackInterface*>(j_native_track)
+      ->RemoveSink(reinterpret_cast<webrtc::AudioTrackSinkInterface*>(j_native_sink));
+}
+
+static jlong JNI_AudioTrack_WrapSink(JNIEnv* jni,
+                                     const JavaParamRef<jobject>& sink) {
+  return jlongFromPointer(new AudioTrackSinkWrapper(jni, sink));
+}
+
+static void JNI_AudioTrack_FreeSink(JNIEnv* jni, jlong j_native_sink) {
+  delete reinterpret_cast<jni::AudioTrackSinkWrapper*>(j_native_sink);
+}
+
+
 }  // namespace jni
 }  // namespace webrtc