diff --git a/build.gradle b/build.gradle index b176711f5d..ff03110089 100644 --- a/build.gradle +++ b/build.gradle @@ -40,6 +40,7 @@ ext { libcameraDriverVersion = "v2026.0.0" rknnVersion = "v2026.0.1" rubikVersion = "dev-v2026.0.1-3-g977bb2e" + ffmpegJniVersion = "dev-v2026.0.0-14-gf6541df" frcYear = "2026" mrcalVersion = "v2026.0.0"; diff --git a/photon-core/build.gradle b/photon-core/build.gradle index a8a3c972f4..f8a286e991 100644 --- a/photon-core/build.gradle +++ b/photon-core/build.gradle @@ -1,3 +1,4 @@ +apply plugin: 'java-library' apply plugin: 'org.photonvision.tools.WpilibTools' import java.nio.file.Path @@ -47,6 +48,15 @@ dependencies { } } + if (jniPlatform == "linuxarm64" || jniPlatform == "linuxx86-64") { + wpilibNatives("org.photonvision:RtspServerJni-jni:$ffmpegJniVersion:$jniPlatform") { + transitive = false + } + } + api("org.photonvision:RtspServerJni-java:$ffmpegJniVersion") { + transitive = false + } + implementation("org.photonvision:rknn_jni-java:$rknnVersion") { transitive = false } diff --git a/photon-core/src/main/java/org/photonvision/common/LoadJNI.java b/photon-core/src/main/java/org/photonvision/common/LoadJNI.java index 154fe40c22..f8dc78e6fb 100644 --- a/photon-core/src/main/java/org/photonvision/common/LoadJNI.java +++ b/photon-core/src/main/java/org/photonvision/common/LoadJNI.java @@ -29,7 +29,8 @@ public enum JNITypes { RUBIK_DETECTOR("tensorflowlite", "tensorflowlite_c", "external_delegate", "rubik_jni"), RKNN_DETECTOR("rga", "rknnrt", "rknn_jni"), MRCAL("mrcal_jni"), - LIBCAMERA("photonlibcamera"); + LIBCAMERA("photonlibcamera"), + FFMPEG("RtspServer"); public final String[] libraries; diff --git a/photon-core/src/main/java/org/photonvision/vision/processes/VisionModule.java b/photon-core/src/main/java/org/photonvision/vision/processes/VisionModule.java index b57b01d969..98dba91db7 100644 --- a/photon-core/src/main/java/org/photonvision/vision/processes/VisionModule.java +++ b/photon-core/src/main/java/org/photonvision/vision/processes/VisionModule.java @@ -42,6 +42,7 @@ import org.photonvision.common.logging.LogGroup; import org.photonvision.common.logging.Logger; import org.photonvision.common.util.SerializationUtils; +import org.photonvision.ffmpeg.FfmpegRtspHandler; import org.photonvision.vision.calibration.CameraCalibrationCoefficients; import org.photonvision.vision.camera.CameraQuirk; import org.photonvision.vision.camera.CameraType; @@ -214,19 +215,19 @@ private void createStreams() { private void recreateStreamResultConsumers() { streamResultConsumers.add( (frame, tgts) -> { - if (frame != null) inputFrameSaver.accept(frame.colorImage); - }); - streamResultConsumers.add( - (frame, tgts) -> { - if (frame != null) outputFrameSaver.accept(frame.processedImage); - }); - streamResultConsumers.add( - (frame, tgts) -> { - if (frame != null) inputVideoStreamer.accept(frame.colorImage); - }); - streamResultConsumers.add( - (frame, tgts) -> { - if (frame != null) outputVideoStreamer.accept(frame.processedImage); + if (frame != null) { + inputFrameSaver.accept(frame.colorImage); + outputFrameSaver.accept(frame.processedImage); + inputVideoStreamer.accept(frame.colorImage); + outputVideoStreamer.accept(frame.processedImage); + + var inputName = visionSource.getSettables().getConfiguration().nickname + "input"; + var outputName = visionSource.getSettables().getConfiguration().nickname + "output"; + // System.out.println(inputName); + FfmpegRtspHandler.putFrame(inputName.toLowerCase(), frame.colorImage.getMat().getNativeObjAddr()); + FfmpegRtspHandler.putFrame( + outputName.toLowerCase(), frame.processedImage.getMat().getNativeObjAddr()); + } }); } diff --git a/photon-server/src/main/java/org/photonvision/Main.java b/photon-server/src/main/java/org/photonvision/Main.java index b07538a587..de5c616421 100644 --- a/photon-server/src/main/java/org/photonvision/Main.java +++ b/photon-server/src/main/java/org/photonvision/Main.java @@ -43,6 +43,7 @@ import org.photonvision.common.logging.PvCSCoreLogger; import org.photonvision.common.networking.NetworkManager; import org.photonvision.common.util.TestUtils; +import org.photonvision.ffmpeg.FfmpegRtspHandler; import org.photonvision.server.Server; import org.photonvision.vision.apriltag.AprilTagFamily; import org.photonvision.vision.calibration.CameraCalibrationCoefficients; @@ -151,10 +152,7 @@ private static void addTestModeSources() { camConf2026 = new CameraConfiguration( PVCameraInfo.fromFileInfo( - TestUtils.getResourcesFolderPath(true) - .resolve("testimages") - .resolve(TestUtils.WPI2026Images.kBlueOutpostFuelSpread.path) - .toString(), + "/home/matt/Downloads/left-cam_input_2025-07-02T065100975_None-0-.jpg", "WPI2026")); camConf2026.FOV = TestUtils.WPI2026Images.FOV.getDegrees(); @@ -174,12 +172,38 @@ private static void addTestModeSources() { double fy = cy / Math.tan(fovHeight.getRadians() / 2.0); JsonMatOfDouble testCameraMatrix = - new JsonMatOfDouble(3, 3, new double[] {fx, 0, cx, 0, fy, cy, 0, 0, 1}); - JsonMatOfDouble testDistortion = new JsonMatOfDouble(1, 5, new double[] {0, 0, 0, 0, 0}); + new JsonMatOfDouble( + 3, + 3, + new double[] { + 688.234790138785, + 0.0, + 395.1728260784305, + 0.0, + 686.4605570601299, + 297.7361999415826, + 0.0, + 0.0, + 1.0 + }); + JsonMatOfDouble testDistortion = + new JsonMatOfDouble( + 1, + 5, + new double[] { + 0.04023968310726899, + -0.05030754420510333, + 8.854257800009915E-4, + -0.001444393564939454, + -0.009338390056925919, + -0.0015440280073886197, + 0.0020340236612642077, + 3.7384608974673787E-4 + }); camConf2026.calibrations.add( new CameraCalibrationCoefficients( - new Size(4000, 1868), + new Size(800, 600), testCameraMatrix, testDistortion, new double[0], @@ -282,6 +306,9 @@ public static void main(String[] args) { System.exit(1); } + tryLoadJNI(JNITypes.FFMPEG); + FfmpegRtspHandler.initialize(); + if (Platform.isRaspberryPi()) { tryLoadJNI(JNITypes.LIBCAMERA); } diff --git a/photon-targeting/src/main/java/org/photonvision/jni/LibraryLoader.java b/photon-targeting/src/main/java/org/photonvision/jni/LibraryLoader.java index 97f686c8b5..68baf31229 100644 --- a/photon-targeting/src/main/java/org/photonvision/jni/LibraryLoader.java +++ b/photon-targeting/src/main/java/org/photonvision/jni/LibraryLoader.java @@ -77,4 +77,16 @@ public static boolean loadTargeting() { } return hasTargetingLoaded; } + + public static boolean loadFfmpeg() { + if (hasTargetingLoaded) return true; + try { + CombinedRuntimeLoader.loadLibraries(LibraryLoader.class, "photontargetingJNI"); + hasTargetingLoaded = true; + } catch (IOException e) { + e.printStackTrace(); + hasTargetingLoaded = false; + } + return hasTargetingLoaded; + } }