diff --git a/composite-pipeline-plan.md b/composite-pipeline-plan.md new file mode 100644 index 0000000000..3527d52457 --- /dev/null +++ b/composite-pipeline-plan.md @@ -0,0 +1,60 @@ +# Composite Pipeline Plan (AprilTag + Object Detection) + +## Goals +- Run AprilTag and Object Detection on the same frame, per camera, without switching pipelines. +- Publish independent results for each detector in the same output format as today. +- Keep the main camera result as a combined list (tags + objects). +- Draw both AprilTag overlays and object detection overlays on the output stream. +- Avoid pre-optimization; match existing object detection behavior on Orange Pi. + +## Outputs (NetworkTables) +- `//result`: combined list (AprilTag + object detections) +- `/-tags/result`: AprilTag-only list +- `/-objects/result`: object-only list + +## Design Overview +- Add a new `Composite` pipeline type. +- Add `CompositePipelineSettings` with: + - AprilTag-specific fields (tagFamily, decimate, blur, threads, refineEdges, decisionMargin, etc.) + - Object detection-specific fields (confidence, nms, model) + - Two toggles: `enableAprilTag`, `enableObjectDetection` + - Shared advanced settings (exposure, solvePNPEnabled, targetModel, output drawing, etc.) +- Implement `CompositePipeline` that: + - Requests `FrameThresholdType.NONE` and uses the color frame directly. + - Builds a reusable grayscale buffer ring for AprilTag detection. + - Runs AprilTag detection and pose estimation. + - Runs object detection. + - Combines targets (tags first, then objects) for the main result. + - Preserves `multiTagResult` and `objectDetectionClassNames`. +- Implement `CompositePipelineResult` to carry split target lists. +- Update NT publishing to publish three `PhotonPipelineResult` streams. +- Update OutputStream drawing to render AprilTags and object detections together. +- Update UI to allow selecting Composite and configuring both AprilTag + Object Detection. + +## Code Touch Points +- `photon-core` + - `vision/pipeline/PipelineType.java` (add Composite) + - `vision/pipeline/CompositePipelineSettings.java` (new) + - `vision/pipeline/CompositePipeline.java` (new) + - `vision/pipeline/result/CompositePipelineResult.java` (new) + - `vision/processes/PipelineManager.java` (create/switch/clone) + - `common/dataflow/networktables/NTDataPublisher.java` (publish combined + split results) + - `vision/pipeline/OutputStreamPipeline.java` (draw both overlays) +- `photon-server` + - `server/DataSocketHandler.java` (pipeline type mapping) +- `photon-client` + - `types/PipelineTypes.ts` (Composite settings + defaults) + - `types/WebsocketDataTypes.ts` (Composite pipeline enum) + - UI components/tabs to show Composite config + +## Risks / Notes +- Combined list changes “best target” semantics on `//result` (tags first by default). +- Composite pipeline runs two detectors per frame; FPS may drop on Orange Pi. + +## Implementation Steps +1. Add Composite pipeline type + settings (Java + TS). +2. Implement CompositePipeline + CompositePipelineResult. +3. Publish split results to `/-tags` and `/-objects`. +4. Update output stream drawing to show both overlays. +5. Update UI to select and configure Composite. +6. Add minimal tests (if feasible) for pipeline creation + NT publishing paths. diff --git a/photon-client/src/components/dashboard/CameraAndPipelineSelectCard.vue b/photon-client/src/components/dashboard/CameraAndPipelineSelectCard.vue index 84df7cd473..cdbd9b991a 100644 --- a/photon-client/src/components/dashboard/CameraAndPipelineSelectCard.vue +++ b/photon-client/src/components/dashboard/CameraAndPipelineSelectCard.vue @@ -32,6 +32,9 @@ const changeCurrentCameraUniqueName = (cameraUniqueName: string) => { case PipelineType.ObjectDetection: pipelineType.value = WebsocketPipelineType.ObjectDetection; break; + case PipelineType.Composite: + pipelineType.value = WebsocketPipelineType.Composite; + break; } }; @@ -138,6 +141,7 @@ const validNewPipelineTypes = computed(() => { ]; if (useSettingsStore().general.supportedBackends.length > 0) { pipelineTypes.push({ name: "Object Detection", value: WebsocketPipelineType.ObjectDetection }); + pipelineTypes.push({ name: "Composite", value: WebsocketPipelineType.Composite }); } return pipelineTypes; }); @@ -176,6 +180,7 @@ const pipelineTypesWrapper = computed<{ name: string; value: number }[]>(() => { ]; if (useSettingsStore().general.supportedBackends.length > 0) { pipelineTypes.push({ name: "Object Detection", value: WebsocketPipelineType.ObjectDetection }); + pipelineTypes.push({ name: "Composite", value: WebsocketPipelineType.Composite }); } if (useCameraSettingsStore().isDriverMode) { @@ -238,6 +243,9 @@ useCameraSettingsStore().$subscribe((mutation, state) => { case PipelineType.ObjectDetection: pipelineType.value = WebsocketPipelineType.ObjectDetection; break; + case PipelineType.Composite: + pipelineType.value = WebsocketPipelineType.Composite; + break; } }); const wrappedCameras = computed(() => diff --git a/photon-client/src/components/dashboard/ConfigOptions.vue b/photon-client/src/components/dashboard/ConfigOptions.vue index be8e017233..9f842abc2d 100644 --- a/photon-client/src/components/dashboard/ConfigOptions.vue +++ b/photon-client/src/components/dashboard/ConfigOptions.vue @@ -89,18 +89,19 @@ const tabGroups = computed(() => { const isAruco = useCameraSettingsStore().currentWebsocketPipelineType === WebsocketPipelineType.Aruco; const isObjectDetection = useCameraSettingsStore().currentWebsocketPipelineType === WebsocketPipelineType.ObjectDetection; + const isComposite = useCameraSettingsStore().currentWebsocketPipelineType === WebsocketPipelineType.Composite; return getTabGroups() .map((tabGroup) => tabGroup.filter( (tabConfig) => !(!allow3d && tabConfig.tabName === "3D") && //Filter out 3D tab any time 3D isn't calibrated - !((!allow3d || isAprilTag || isAruco || isObjectDetection) && tabConfig.tabName === "PnP") && //Filter out the PnP config tab if 3D isn't available, or we're doing AprilTags - !((isAprilTag || isAruco || isObjectDetection) && tabConfig.tabName === "Threshold") && //Filter out threshold tab if we're doing AprilTags - !((isAprilTag || isAruco || isObjectDetection) && tabConfig.tabName === "Contours") && //Filter out contours if we're doing AprilTags - !(!isAprilTag && tabConfig.tabName === "AprilTag") && //Filter out apriltag unless we actually are doing AprilTags + !((!allow3d || isAprilTag || isAruco || isObjectDetection || isComposite) && tabConfig.tabName === "PnP") && //Filter out the PnP config tab if 3D isn't available, or we're doing AprilTags + !((isAprilTag || isAruco || isObjectDetection || isComposite) && tabConfig.tabName === "Threshold") && //Filter out threshold tab if we're doing AprilTags + !((isAprilTag || isAruco || isObjectDetection || isComposite) && tabConfig.tabName === "Contours") && //Filter out contours if we're doing AprilTags + !(!(isAprilTag || isComposite) && tabConfig.tabName === "AprilTag") && //Filter out apriltag unless we actually are doing AprilTags !(!isAruco && tabConfig.tabName === "ArUco") && - !(!isObjectDetection && tabConfig.tabName === "Object Detection") //Filter out ArUco unless we actually are doing ArUco + !(!(isObjectDetection || isComposite) && tabConfig.tabName === "Object Detection") //Filter out ArUco unless we actually are doing ArUco ) ) .filter((it) => it.length); // Remove empty tab groups diff --git a/photon-client/src/components/dashboard/tabs/AprilTagTab.vue b/photon-client/src/components/dashboard/tabs/AprilTagTab.vue index 249f80607e..1251d244c7 100644 --- a/photon-client/src/components/dashboard/tabs/AprilTagTab.vue +++ b/photon-client/src/components/dashboard/tabs/AprilTagTab.vue @@ -1,32 +1,51 @@