Skip to content

Native H.264 decoding using FFMPEG over CLI/IPC #7658

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
emilk wants to merge 13 commits into from
Closed

Native H.264 decoding using FFMPEG over CLI/IPC #7658

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
c5b3b3a
Initial ffmpeg H.264 support in frames example
emilk Oct 8, 2024
d4da02a
Actual H.264 video playback inside of Rerun viewer
emilk Oct 8, 2024
f2fc70b
Fix timestamps and seeking
emilk Oct 9, 2024
c07b015
Code cleanup and better error handling
emilk Oct 9, 2024
f08e5e6
Better error handling
emilk Oct 9, 2024
b2b842b
Improve log output and thread names
emilk Oct 9, 2024
26f4834
Fix ffmpeg feature flag
emilk Oct 9, 2024
a6d81a8
Merge branch 'main' into emilk/ffmpeg-decoder
emilk Oct 17, 2024
cd734b4
Merge remote-tracking branch 'origin/main' into emilk/ffmpeg-decoder
Wumpf Oct 25, 2024
19f01ad
reduce ffmpeg decode delay
Wumpf Oct 25, 2024
d0c9e6b
fix re_video example build
Wumpf Oct 25, 2024
cb6d4d1
add decode timestamp, nal header parsing, various comments, todo notes
Wumpf Oct 25, 2024
6391b85
better timestamp syncing strategy?
Wumpf Oct 25, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions Cargo.lock
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2399,6 +2399,15 @@ dependencies = [
"simd-adler32",
]

[[package]]
name = "ffmpeg-sidecar"
version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4bd1e249e0ceeb0f5c9f84a3c6941c3bde3ebc2815f4b94531a7e806af61c4c0"
dependencies = [
"anyhow",
]

[[package]]
name = "filetime"
version = "0.2.23"
Expand Down Expand Up @@ -6183,6 +6192,7 @@ dependencies = [
"criterion",
"crossbeam",
"econtext",
"ffmpeg-sidecar",
"indicatif",
"itertools 0.13.0",
"parking_lot",
Expand Down
1 change: 1 addition & 0 deletions Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -176,6 +176,7 @@ econtext = "0.2" # Prints error contexts on crashes
ehttp = "0.5.0"
enumset = "1.0.12"
env_logger = { version = "0.10", default-features = false }
ffmpeg-sidecar = "1.1.2"
fixed = { version = "<1.28", default-features = false } # 1.28+ is MSRV 1.79+
flatbuffers = "23.0"
futures-channel = "0.3"
Expand Down
9 changes: 8 additions & 1 deletion crates/store/re_video/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,11 +23,14 @@ features = ["all"]


[features]
default = ["av1"]
default = ["av1", "ffmpeg"]

## Native AV1 decoding.
av1 = ["dep:dav1d"]

## Decode H.264 using ffmpeg over CLI.
ffmpeg = ["dep:ffmpeg-sidecar"]

## Enable faster native video decoding with assembly.
## You need to install [nasm](https://nasm.us/) to compile with this feature.
nasm = [
Expand Down Expand Up @@ -61,7 +64,11 @@ dav1d = { workspace = true, optional = true, default-features = false, features
"bitdepth_8",
] }

ffmpeg-sidecar = { workspace = true, optional = true }


[dev-dependencies]
# For the `frames` example:
indicatif.workspace = true
criterion.workspace = true

Expand Down
35 changes: 32 additions & 3 deletions crates/store/re_video/examples/frames.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@ use indicatif::ProgressBar;
use parking_lot::Mutex;

fn main() {
re_log::setup_logging();

// frames <video.mp4>
let args: Vec<_> = std::env::args().collect();
let Some(video_path) = args.get(1) else {
Expand Down Expand Up @@ -88,8 +90,18 @@ fn write_video_frames(
.create(true)
.truncate(true)
.open(output_dir.join(format!("{i:0width$}.ppm")))
.expect("failed to open file");
write_binary_ppm(&mut file, frame.width, frame.height, &frame.data);
.expect("failed to oformatpen file");
match frame.format {
re_video::PixelFormat::Rgb8Unorm => {
write_ppm_rgb24(&mut file, frame.width, frame.height, &frame.data);
}
re_video::PixelFormat::Rgba8Unorm => {
write_ppm_rgba32(&mut file, frame.width, frame.height, &frame.data);
}
re_video::PixelFormat::Yuv { .. } => {
re_log::error_once!("YUV frame writing is not not supported");
}
}
}
}
}
Expand All @@ -98,7 +110,24 @@ fn num_digits(n: usize) -> usize {
(n as f64).log10().floor() as usize + 1
}

fn write_binary_ppm(file: &mut File, width: u32, height: u32, rgba: &[u8]) {
fn write_ppm_rgb24(file: &mut File, width: u32, height: u32, rgb: &[u8]) {
assert_eq!(width as usize * height as usize * 3, rgb.len());

let header = format!("P6\n{width} {height}\n255\n");

let mut data = Vec::with_capacity(header.len() + width as usize * height as usize * 3);
data.extend_from_slice(header.as_bytes());

for rgb in rgb.chunks(3) {
data.extend_from_slice(&[rgb[0], rgb[1], rgb[2]]);
}

file.write_all(&data).expect("failed to write frame data");
}

fn write_ppm_rgba32(file: &mut File, width: u32, height: u32, rgba: &[u8]) {
assert_eq!(width as usize * height as usize * 4, rgba.len());

let header = format!("P6\n{width} {height}\n255\n");

let mut data = Vec::with_capacity(header.len() + width as usize * height as usize * 3);
Expand Down
2 changes: 1 addition & 1 deletion crates/store/re_video/src/decode/async_decoder.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ impl AsyncDecoder {
let comms = Comms::default();

let thread = std::thread::Builder::new()
.name("av1_decoder".into())
.name(format!("decoer thread for {debug_name}"))
.spawn({
let comms = comms.clone();
move || {
Expand Down
2 changes: 1 addition & 1 deletion crates/store/re_video/src/decode/av1.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -232,7 +232,7 @@ fn output_picture(
width: picture.width(),
height: picture.height(),
format,
timestamp: Time(picture.timestamp().unwrap_or(0)),
composition_timestamp: Time(picture.timestamp().unwrap_or(0)),
duration: Time(picture.duration()),
};
on_output(Ok(frame));
Expand Down
Loading