Skip to content

Commit 97d1d22

Browse files
jkurijkuri
committed
feat(audio): cache denoised mic audio stream so it doesn't regenerates everytime you open the project, also use the cached stream when exporting
1 parent 76efdb7 commit 97d1d22

File tree

5 files changed

+120
-30
lines changed

5 files changed

+120
-30
lines changed

‎Reframed/Compositor/VideoCompositor.swift‎

Lines changed: 18 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ enum VideoCompositor {
4646
micAudioVolume: Float = 1.0,
4747
micNoiseReductionEnabled: Bool = false,
4848
micNoiseReductionIntensity: Float = 0.5,
49+
processedMicAudioURL: URL? = nil,
4950
progressHandler: (@MainActor @Sendable (Double, Double?) -> Void)? = nil
5051
) async throws -> URL {
5152
let composition = AVMutableComposition()
@@ -72,18 +73,26 @@ enum VideoCompositor {
7273
try compScreenTrack?.insertTimeRange(effectiveTrim, of: screenVideoTrack, at: .zero)
7374

7475
var processedMicURL: URL?
76+
var shouldCleanupProcessedMic = false
7577
if let micURL = result.microphoneAudioURL, micNoiseReductionEnabled, micAudioVolume > 0 {
76-
let tempURL = FileManager.default.temporaryDirectory
77-
.appendingPathComponent("reframed-nr-\(UUID().uuidString).m4a")
78-
try await RNNoiseProcessor.processFile(
79-
inputURL: micURL,
80-
outputURL: tempURL,
81-
intensity: micNoiseReductionIntensity
82-
)
83-
processedMicURL = tempURL
78+
if let cachedURL = processedMicAudioURL, FileManager.default.fileExists(atPath: cachedURL.path) {
79+
processedMicURL = cachedURL
80+
} else {
81+
let tempURL = FileManager.default.temporaryDirectory
82+
.appendingPathComponent("reframed-nr-\(UUID().uuidString).m4a")
83+
try await RNNoiseProcessor.processFile(
84+
inputURL: micURL,
85+
outputURL: tempURL,
86+
intensity: micNoiseReductionIntensity
87+
)
88+
processedMicURL = tempURL
89+
shouldCleanupProcessedMic = true
90+
}
8491
}
8592
defer {
86-
if let url = processedMicURL { try? FileManager.default.removeItem(at: url) }
93+
if shouldCleanupProcessedMic, let url = processedMicURL {
94+
try? FileManager.default.removeItem(at: url)
95+
}
8796
}
8897

8998
var audioSources: [AudioSource] = []

‎Reframed/Editor/EditorState.swift‎

Lines changed: 76 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,7 @@ final class EditorState {
8484
var exportProgress: Double = 0
8585
var exportETA: Double?
8686
var exportTask: Task<Void, Never>?
87+
var exportStatusMessage: String?
8788
var isPreviewMode = false
8889

8990
var backgroundStyle: BackgroundStyle = .solidColor(CodableColor(r: 0, g: 0, b: 0))
@@ -463,7 +464,9 @@ final class EditorState {
463464
micProcessingTask?.cancel()
464465
guard let micURL = result.microphoneAudioURL, micNoiseReductionEnabled else {
465466
if let old = processedMicAudioURL {
466-
try? FileManager.default.removeItem(at: old)
467+
if !isURLInsideProjectBundle(old) {
468+
try? FileManager.default.removeItem(at: old)
469+
}
467470
processedMicAudioURL = nil
468471
}
469472
isMicProcessing = false
@@ -473,9 +476,25 @@ final class EditorState {
473476
return
474477
}
475478

479+
let intensity = micNoiseReductionIntensity
480+
481+
if let proj = project,
482+
let cachedURL = proj.denoisedMicAudioURL,
483+
let cachedIntensity = proj.metadata.editorState?.audioSettings?.cachedNoiseReductionIntensity,
484+
abs(cachedIntensity - intensity) < 0.001
485+
{
486+
let oldURL = processedMicAudioURL
487+
processedMicAudioURL = cachedURL
488+
isMicProcessing = false
489+
playerController.swapMicAudioFile(url: cachedURL)
490+
if let oldURL, oldURL != cachedURL, !isURLInsideProjectBundle(oldURL) {
491+
try? FileManager.default.removeItem(at: oldURL)
492+
}
493+
return
494+
}
495+
476496
isMicProcessing = true
477497
micProcessingProgress = 0
478-
let intensity = micNoiseReductionIntensity
479498
let state = self
480499
micProcessingTask = Task {
481500
try? await Task.sleep(for: .milliseconds(500))
@@ -497,22 +516,44 @@ final class EditorState {
497516
try? FileManager.default.removeItem(at: tempURL)
498517
return
499518
}
500-
let oldURL = processedMicAudioURL
501-
processedMicAudioURL = tempURL
502-
isMicProcessing = false
503-
playerController.swapMicAudioFile(url: tempURL)
504-
if let oldURL { try? FileManager.default.removeItem(at: oldURL) }
519+
520+
var finalURL = tempURL
521+
if let proj = state.project {
522+
let destURL = proj.denoisedMicAudioDestinationURL
523+
try? FileManager.default.removeItem(at: destURL)
524+
do {
525+
try FileManager.default.copyItem(at: tempURL, to: destURL)
526+
try? FileManager.default.removeItem(at: tempURL)
527+
finalURL = destURL
528+
} catch {
529+
state.logger.warning("Failed to cache denoised audio in bundle: \(error)")
530+
}
531+
}
532+
533+
let oldURL = state.processedMicAudioURL
534+
state.processedMicAudioURL = finalURL
535+
state.isMicProcessing = false
536+
state.playerController.swapMicAudioFile(url: finalURL)
537+
if let oldURL, oldURL != finalURL, !state.isURLInsideProjectBundle(oldURL) {
538+
try? FileManager.default.removeItem(at: oldURL)
539+
}
540+
state.scheduleSave()
505541
} catch {
506542
guard !Task.isCancelled else {
507543
try? FileManager.default.removeItem(at: tempURL)
508544
return
509545
}
510-
isMicProcessing = false
511-
logger.error("Mic noise reduction failed: \(error)")
546+
state.isMicProcessing = false
547+
state.logger.error("Mic noise reduction failed: \(error)")
512548
}
513549
}
514550
}
515551

552+
private func isURLInsideProjectBundle(_ url: URL) -> Bool {
553+
guard let bundleURL = project?.bundleURL else { return false }
554+
return url.path.hasPrefix(bundleURL.path)
555+
}
556+
516557
func isCameraFullscreen(at time: Double) -> Bool {
517558
cameraFullscreenRegions.contains { time >= $0.startSeconds && time <= $0.endSeconds }
518559
}
@@ -646,9 +687,22 @@ final class EditorState {
646687
isExporting = true
647688
exportProgress = 0
648689
exportETA = nil
690+
exportStatusMessage = nil
649691
defer {
650692
isExporting = false
651693
exportTask = nil
694+
exportStatusMessage = nil
695+
}
696+
697+
if isMicProcessing {
698+
exportStatusMessage =
699+
"Waiting for noise reduction… \(Int(micProcessingProgress * 100))%"
700+
while isMicProcessing {
701+
try await Task.sleep(for: .milliseconds(100))
702+
exportStatusMessage =
703+
"Waiting for noise reduction… \(Int(micProcessingProgress * 100))%"
704+
}
705+
exportStatusMessage = nil
652706
}
653707

654708
let cursorSnapshot = showCursor ? activeCursorProvider?.makeSnapshot() : nil
@@ -722,6 +776,7 @@ final class EditorState {
722776
micAudioVolume: effectiveMicAudioVolume,
723777
micNoiseReductionEnabled: micNoiseReductionEnabled,
724778
micNoiseReductionIntensity: micNoiseReductionIntensity,
779+
processedMicAudioURL: processedMicAudioURL,
725780
progressHandler: { progress, eta in
726781
state.exportProgress = progress
727782
state.exportETA = eta
@@ -953,13 +1008,21 @@ final class EditorState {
9531008
}
9541009
var audioSettings: AudioSettingsData?
9551010
if hasSystemAudio || hasMicAudio {
1011+
var cachedIntensity: Float?
1012+
if micNoiseReductionEnabled,
1013+
let proj = project,
1014+
proj.denoisedMicAudioURL != nil
1015+
{
1016+
cachedIntensity = micNoiseReductionIntensity
1017+
}
9561018
audioSettings = AudioSettingsData(
9571019
systemAudioVolume: systemAudioVolume,
9581020
micAudioVolume: micAudioVolume,
9591021
systemAudioMuted: systemAudioMuted,
9601022
micAudioMuted: micAudioMuted,
9611023
micNoiseReductionEnabled: micNoiseReductionEnabled,
962-
micNoiseReductionIntensity: micNoiseReductionIntensity
1024+
micNoiseReductionIntensity: micNoiseReductionIntensity,
1025+
cachedNoiseReductionIntensity: cachedIntensity
9631026
)
9641027
}
9651028
return EditorStateData(
@@ -1212,7 +1275,9 @@ final class EditorState {
12121275
}
12131276
playerController.teardown()
12141277
if let url = processedMicAudioURL {
1215-
try? FileManager.default.removeItem(at: url)
1278+
if !isURLInsideProjectBundle(url) {
1279+
try? FileManager.default.removeItem(at: url)
1280+
}
12161281
processedMicAudioURL = nil
12171282
}
12181283
}

‎Reframed/Editor/EditorTopBar.swift‎

Lines changed: 16 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -10,17 +10,23 @@ struct EditorTopBar: View {
1010
let _ = colorScheme
1111
ZStack {
1212
if editorState.isExporting {
13-
HStack(spacing: 8) {
14-
ProgressView(value: editorState.exportProgress)
15-
.frame(width: 200)
16-
Text("\(Int(editorState.exportProgress * 100))%")
17-
.font(.system(size: 11).monospacedDigit())
13+
if let statusMessage = editorState.exportStatusMessage {
14+
Text(statusMessage)
15+
.font(.system(size: 12, weight: .medium).monospacedDigit())
1816
.foregroundStyle(ReframedColors.secondaryText)
19-
.frame(width: 32, alignment: .trailing)
20-
Text(editorState.exportETA.map { $0 > 0 ? "ETA \(formatDuration(seconds: Int(ceil($0))))" : "" } ?? "")
21-
.font(.system(size: 11).monospacedDigit())
22-
.foregroundStyle(ReframedColors.secondaryText)
23-
.frame(width: 72, alignment: .leading)
17+
} else {
18+
HStack(spacing: 8) {
19+
ProgressView(value: editorState.exportProgress)
20+
.frame(width: 200)
21+
Text("\(Int(editorState.exportProgress * 100))%")
22+
.font(.system(size: 11).monospacedDigit())
23+
.foregroundStyle(ReframedColors.secondaryText)
24+
.frame(width: 32, alignment: .trailing)
25+
Text(editorState.exportETA.map { $0 > 0 ? "ETA \(formatDuration(seconds: Int(ceil($0))))" : "" } ?? "")
26+
.font(.system(size: 11).monospacedDigit())
27+
.foregroundStyle(ReframedColors.secondaryText)
28+
.frame(width: 72, alignment: .leading)
29+
}
2430
}
2531
} else {
2632
Text(editorState.projectName)

‎Reframed/Project/ProjectMetadata.swift‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ struct AudioSettingsData: Codable, Sendable, Equatable {
4747
var micAudioMuted: Bool = false
4848
var micNoiseReductionEnabled: Bool = false
4949
var micNoiseReductionIntensity: Float = 0.5
50+
var cachedNoiseReductionIntensity: Float?
5051
}
5152

5253
struct AudioRegionData: Codable, Sendable, Identifiable, Equatable {

‎Reframed/Project/ReframedProject.swift‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,15 @@ struct ReframedProject: Sendable {
2828
return FileManager.default.fileExists(atPath: url.path) ? url : nil
2929
}
3030

31+
var denoisedMicAudioURL: URL? {
32+
let url = bundleURL.appendingPathComponent("denoised-mic.m4a")
33+
return FileManager.default.fileExists(atPath: url.path) ? url : nil
34+
}
35+
36+
var denoisedMicAudioDestinationURL: URL {
37+
bundleURL.appendingPathComponent("denoised-mic.m4a")
38+
}
39+
3140
var cursorMetadataURL: URL? {
3241
let url = bundleURL.appendingPathComponent("cursor-metadata.json")
3342
return FileManager.default.fileExists(atPath: url.path) ? url : nil

0 commit comments

Comments
 (0)