Ffmpeg HEVC + RTSP Server

build.gradle

@@ -40,6 +40,7 @@ ext {
     libcameraDriverVersion = "v2026.0.0"
     rknnVersion = "v2026.0.1"
     rubikVersion = "dev-v2026.0.1-3-g977bb2e"
+    ffmpegJniVersion = "dev-v2026.0.0-14-gf6541df"
     frcYear = "2026"
     mrcalVersion = "v2026.0.0";
 

photon-core/build.gradle

@@ -1,3 +1,4 @@
+apply plugin: 'java-library'
 apply plugin: 'org.photonvision.tools.WpilibTools'
 
 import java.nio.file.Path
@@ -47,6 +48,15 @@ dependencies {
         }
     }
 
+    if (jniPlatform == "linuxarm64" || jniPlatform == "linuxx86-64") {
+        wpilibNatives("org.photonvision:RtspServerJni-jni:$ffmpegJniVersion:$jniPlatform") {
+            transitive = false
+        }
+    }
+    api("org.photonvision:RtspServerJni-java:$ffmpegJniVersion") {
+        transitive = false
+    }
+
     implementation("org.photonvision:rknn_jni-java:$rknnVersion") {
         transitive = false
     }

photon-core/src/main/java/org/photonvision/common/LoadJNI.java

@@ -29,7 +29,8 @@ public enum JNITypes {
         RUBIK_DETECTOR("tensorflowlite", "tensorflowlite_c", "external_delegate", "rubik_jni"),
         RKNN_DETECTOR("rga", "rknnrt", "rknn_jni"),
         MRCAL("mrcal_jni"),
-        LIBCAMERA("photonlibcamera");
+        LIBCAMERA("photonlibcamera"),
+        FFMPEG("RtspServer");
 
         public final String[] libraries;
 

photon-core/src/main/java/org/photonvision/vision/processes/VisionModule.java

@@ -42,6 +42,7 @@
 import org.photonvision.common.logging.LogGroup;
 import org.photonvision.common.logging.Logger;
 import org.photonvision.common.util.SerializationUtils;
+import org.photonvision.ffmpeg.FfmpegRtspHandler;
 import org.photonvision.vision.calibration.CameraCalibrationCoefficients;
 import org.photonvision.vision.camera.CameraQuirk;
 import org.photonvision.vision.camera.CameraType;
@@ -214,19 +215,19 @@ private void createStreams() {
     private void recreateStreamResultConsumers() {
         streamResultConsumers.add(
                 (frame, tgts) -> {
-                    if (frame != null) inputFrameSaver.accept(frame.colorImage);
-                });
-        streamResultConsumers.add(
-                (frame, tgts) -> {
-                    if (frame != null) outputFrameSaver.accept(frame.processedImage);
-                });
-        streamResultConsumers.add(
-                (frame, tgts) -> {
-                    if (frame != null) inputVideoStreamer.accept(frame.colorImage);
-                });
-        streamResultConsumers.add(
-                (frame, tgts) -> {
-                    if (frame != null) outputVideoStreamer.accept(frame.processedImage);
+                    if (frame != null) {
+                        inputFrameSaver.accept(frame.colorImage);
+                        outputFrameSaver.accept(frame.processedImage);
+                        inputVideoStreamer.accept(frame.colorImage);
+                        outputVideoStreamer.accept(frame.processedImage);
+
+                        var inputName = visionSource.getSettables().getConfiguration().nickname + "input";
+                        var outputName = visionSource.getSettables().getConfiguration().nickname + "output";
+                        // System.out.println(inputName);
+                        FfmpegRtspHandler.putFrame(inputName.toLowerCase(), frame.colorImage.getMat().getNativeObjAddr());
+                        FfmpegRtspHandler.putFrame(
+                                outputName.toLowerCase(), frame.processedImage.getMat().getNativeObjAddr());
+                    }
                 });
     }
 

photon-server/src/main/java/org/photonvision/Main.java

@@ -43,6 +43,7 @@
 import org.photonvision.common.logging.PvCSCoreLogger;
 import org.photonvision.common.networking.NetworkManager;
 import org.photonvision.common.util.TestUtils;
+import org.photonvision.ffmpeg.FfmpegRtspHandler;
 import org.photonvision.server.Server;
 import org.photonvision.vision.apriltag.AprilTagFamily;
 import org.photonvision.vision.calibration.CameraCalibrationCoefficients;
@@ -151,10 +152,7 @@ private static void addTestModeSources() {
             camConf2026 =
                     new CameraConfiguration(
                             PVCameraInfo.fromFileInfo(
-                                    TestUtils.getResourcesFolderPath(true)
-                                            .resolve("testimages")
-                                            .resolve(TestUtils.WPI2026Images.kBlueOutpostFuelSpread.path)
-                                            .toString(),
+                                    "/home/matt/Downloads/left-cam_input_2025-07-02T065100975_None-0-.jpg",
                                     "WPI2026"));
 
             camConf2026.FOV = TestUtils.WPI2026Images.FOV.getDegrees();
@@ -174,12 +172,38 @@ private static void addTestModeSources() {
             double fy = cy / Math.tan(fovHeight.getRadians() / 2.0);
 
             JsonMatOfDouble testCameraMatrix =
-                    new JsonMatOfDouble(3, 3, new double[] {fx, 0, cx, 0, fy, cy, 0, 0, 1});
-            JsonMatOfDouble testDistortion = new JsonMatOfDouble(1, 5, new double[] {0, 0, 0, 0, 0});
+                    new JsonMatOfDouble(
+                            3,
+                            3,
+                            new double[] {
+                                688.234790138785,
+                                0.0,
+                                395.1728260784305,
+                                0.0,
+                                686.4605570601299,
+                                297.7361999415826,
+                                0.0,
+                                0.0,
+                                1.0
+                            });
+            JsonMatOfDouble testDistortion =
+                    new JsonMatOfDouble(
+                            1,
+                            5,
+                            new double[] {
+                                0.04023968310726899,
+                                -0.05030754420510333,
+                                8.854257800009915E-4,
+                                -0.001444393564939454,
+                                -0.009338390056925919,
+                                -0.0015440280073886197,
+                                0.0020340236612642077,
+                                3.7384608974673787E-4
+                            });
 
             camConf2026.calibrations.add(
                     new CameraCalibrationCoefficients(
-                            new Size(4000, 1868),
+                            new Size(800, 600),
                             testCameraMatrix,
                             testDistortion,
                             new double[0],
@@ -282,6 +306,9 @@ public static void main(String[] args) {
             System.exit(1);
         }
 
+        tryLoadJNI(JNITypes.FFMPEG);
+        FfmpegRtspHandler.initialize();
+
         if (Platform.isRaspberryPi()) {
             tryLoadJNI(JNITypes.LIBCAMERA);
         }

photon-targeting/src/main/java/org/photonvision/jni/LibraryLoader.java

@@ -77,4 +77,16 @@ public static boolean loadTargeting() {
         }
         return hasTargetingLoaded;
     }
+
+    public static boolean loadFfmpeg() {
+        if (hasTargetingLoaded) return true;
+        try {
+            CombinedRuntimeLoader.loadLibraries(LibraryLoader.class, "photontargetingJNI");
+            hasTargetingLoaded = true;
+        } catch (IOException e) {
+            e.printStackTrace();
+            hasTargetingLoaded = false;
+        }
+        return hasTargetingLoaded;
+    }
 }