Add composite pipeline for tags and object detection

composite-pipeline-plan.md

@@ -0,0 +1,60 @@
+# Composite Pipeline Plan (AprilTag + Object Detection)
+
+## Goals
+- Run AprilTag and Object Detection on the same frame, per camera, without switching pipelines.
+- Publish independent results for each detector in the same output format as today.
+- Keep the main camera result as a combined list (tags + objects).
+- Draw both AprilTag overlays and object detection overlays on the output stream.
+- Avoid pre-optimization; match existing object detection behavior on Orange Pi.
+
+## Outputs (NetworkTables)
+- `/<camera>/result`: combined list (AprilTag + object detections)
+- `/<camera>-tags/result`: AprilTag-only list
+- `/<camera>-objects/result`: object-only list
+
+## Design Overview
+- Add a new `Composite` pipeline type.
+- Add `CompositePipelineSettings` with:
+  - AprilTag-specific fields (tagFamily, decimate, blur, threads, refineEdges, decisionMargin, etc.)
+  - Object detection-specific fields (confidence, nms, model)
+  - Two toggles: `enableAprilTag`, `enableObjectDetection`
+  - Shared advanced settings (exposure, solvePNPEnabled, targetModel, output drawing, etc.)
+- Implement `CompositePipeline` that:
+  - Requests `FrameThresholdType.NONE` and uses the color frame directly.
+  - Builds a reusable grayscale buffer ring for AprilTag detection.
+  - Runs AprilTag detection and pose estimation.
+  - Runs object detection.
+  - Combines targets (tags first, then objects) for the main result.
+  - Preserves `multiTagResult` and `objectDetectionClassNames`.
+- Implement `CompositePipelineResult` to carry split target lists.
+- Update NT publishing to publish three `PhotonPipelineResult` streams.
+- Update OutputStream drawing to render AprilTags and object detections together.
+- Update UI to allow selecting Composite and configuring both AprilTag + Object Detection.
+
+## Code Touch Points
+- `photon-core`
+  - `vision/pipeline/PipelineType.java` (add Composite)
+  - `vision/pipeline/CompositePipelineSettings.java` (new)
+  - `vision/pipeline/CompositePipeline.java` (new)
+  - `vision/pipeline/result/CompositePipelineResult.java` (new)
+  - `vision/processes/PipelineManager.java` (create/switch/clone)
+  - `common/dataflow/networktables/NTDataPublisher.java` (publish combined + split results)
+  - `vision/pipeline/OutputStreamPipeline.java` (draw both overlays)
+- `photon-server`
+  - `server/DataSocketHandler.java` (pipeline type mapping)
+- `photon-client`
+  - `types/PipelineTypes.ts` (Composite settings + defaults)
+  - `types/WebsocketDataTypes.ts` (Composite pipeline enum)
+  - UI components/tabs to show Composite config
+
+## Risks / Notes
+- Combined list changes “best target” semantics on `/<camera>/result` (tags first by default).
+- Composite pipeline runs two detectors per frame; FPS may drop on Orange Pi.
+
+## Implementation Steps
+1. Add Composite pipeline type + settings (Java + TS).
+2. Implement CompositePipeline + CompositePipelineResult.
+3. Publish split results to `/<camera>-tags` and `/<camera>-objects`.
+4. Update output stream drawing to show both overlays.
+5. Update UI to select and configure Composite.
+6. Add minimal tests (if feasible) for pipeline creation + NT publishing paths.

photon-client/src/components/dashboard/CameraAndPipelineSelectCard.vue

@@ -32,6 +32,9 @@ const changeCurrentCameraUniqueName = (cameraUniqueName: string) => {
     case PipelineType.ObjectDetection:
       pipelineType.value = WebsocketPipelineType.ObjectDetection;
       break;
+    case PipelineType.Composite:
+      pipelineType.value = WebsocketPipelineType.Composite;
+      break;
   }
 };
 
@@ -138,6 +141,7 @@ const validNewPipelineTypes = computed(() => {
   ];
   if (useSettingsStore().general.supportedBackends.length > 0) {
     pipelineTypes.push({ name: "Object Detection", value: WebsocketPipelineType.ObjectDetection });
+    pipelineTypes.push({ name: "Composite", value: WebsocketPipelineType.Composite });
   }
   return pipelineTypes;
 });
@@ -176,6 +180,7 @@ const pipelineTypesWrapper = computed<{ name: string; value: number }[]>(() => {
   ];
   if (useSettingsStore().general.supportedBackends.length > 0) {
     pipelineTypes.push({ name: "Object Detection", value: WebsocketPipelineType.ObjectDetection });
+    pipelineTypes.push({ name: "Composite", value: WebsocketPipelineType.Composite });
   }
 
   if (useCameraSettingsStore().isDriverMode) {
@@ -238,6 +243,9 @@ useCameraSettingsStore().$subscribe((mutation, state) => {
     case PipelineType.ObjectDetection:
       pipelineType.value = WebsocketPipelineType.ObjectDetection;
       break;
+    case PipelineType.Composite:
+      pipelineType.value = WebsocketPipelineType.Composite;
+      break;
   }
 });
 const wrappedCameras = computed<SelectItem[]>(() =>

photon-client/src/components/dashboard/ConfigOptions.vue

@@ -89,18 +89,19 @@ const tabGroups = computed<ConfigOption[][]>(() => {
   const isAruco = useCameraSettingsStore().currentWebsocketPipelineType === WebsocketPipelineType.Aruco;
   const isObjectDetection =
     useCameraSettingsStore().currentWebsocketPipelineType === WebsocketPipelineType.ObjectDetection;
+  const isComposite = useCameraSettingsStore().currentWebsocketPipelineType === WebsocketPipelineType.Composite;
 
   return getTabGroups()
     .map((tabGroup) =>
       tabGroup.filter(
         (tabConfig) =>
           !(!allow3d && tabConfig.tabName === "3D") && //Filter out 3D tab any time 3D isn't calibrated
-          !((!allow3d || isAprilTag || isAruco || isObjectDetection) && tabConfig.tabName === "PnP") && //Filter out the PnP config tab if 3D isn't available, or we're doing AprilTags
-          !((isAprilTag || isAruco || isObjectDetection) && tabConfig.tabName === "Threshold") && //Filter out threshold tab if we're doing AprilTags
-          !((isAprilTag || isAruco || isObjectDetection) && tabConfig.tabName === "Contours") && //Filter out contours if we're doing AprilTags
-          !(!isAprilTag && tabConfig.tabName === "AprilTag") && //Filter out apriltag unless we actually are doing AprilTags
+          !((!allow3d || isAprilTag || isAruco || isObjectDetection || isComposite) && tabConfig.tabName === "PnP") && //Filter out the PnP config tab if 3D isn't available, or we're doing AprilTags
+          !((isAprilTag || isAruco || isObjectDetection || isComposite) && tabConfig.tabName === "Threshold") && //Filter out threshold tab if we're doing AprilTags
+          !((isAprilTag || isAruco || isObjectDetection || isComposite) && tabConfig.tabName === "Contours") && //Filter out contours if we're doing AprilTags
+          !(!(isAprilTag || isComposite) && tabConfig.tabName === "AprilTag") && //Filter out apriltag unless we actually are doing AprilTags
           !(!isAruco && tabConfig.tabName === "ArUco") &&
-          !(!isObjectDetection && tabConfig.tabName === "Object Detection") //Filter out ArUco unless we actually are doing ArUco
+          !(!(isObjectDetection || isComposite) && tabConfig.tabName === "Object Detection") //Filter out ArUco unless we actually are doing ArUco
       )
     )
     .filter((it) => it.length); // Remove empty tab groups

photon-client/src/components/dashboard/tabs/AprilTagTab.vue

@@ -1,32 +1,51 @@
 <script setup lang="ts">
-import { PipelineType } from "@/types/PipelineTypes";
+import { type AprilTagPipelineSettings, type CompositePipelineSettings, PipelineType } from "@/types/PipelineTypes";
 import PvSelect from "@/components/common/pv-select.vue";
 import PvSlider from "@/components/common/pv-slider.vue";
 import PvSwitch from "@/components/common/pv-switch.vue";
 import { computed } from "vue";
 import { useStateStore } from "@/stores/StateStore";
-import type { ActivePipelineSettings } from "@/types/PipelineTypes";
 import { useCameraSettingsStore } from "@/stores/settings/CameraSettingsStore";
 import { useDisplay } from "vuetify";
 
 // TODO fix pipeline typing in order to fix this, the store settings call should be able to infer that only valid pipeline type settings are exposed based on pre-checks for the entire config section
 // Defer reference to store access method
-const currentPipelineSettings = computed<ActivePipelineSettings>(
-  () => useCameraSettingsStore().currentPipelineSettings
+const currentPipelineSettings = computed<AprilTagPipelineSettings | CompositePipelineSettings>(
+  () => useCameraSettingsStore().currentPipelineSettings as AprilTagPipelineSettings
 );
 const { mdAndDown } = useDisplay();
 const interactiveCols = computed(() =>
   mdAndDown.value && (!useStateStore().sidebarFolded || useCameraSettingsStore().isDriverMode) ? 8 : 7
 );
+const aprilTagDisabled = computed(
+  () =>
+    currentPipelineSettings.value.pipelineType === PipelineType.Composite &&
+    !currentPipelineSettings.value.enableAprilTag
+);
 </script>
 
 <template>
-  <div v-if="currentPipelineSettings.pipelineType === PipelineType.AprilTag">
+  <div
+    v-if="
+      currentPipelineSettings.pipelineType === PipelineType.AprilTag ||
+      currentPipelineSettings.pipelineType === PipelineType.Composite
+    "
+  >
+    <pv-switch
+      v-if="currentPipelineSettings.pipelineType === PipelineType.Composite"
+      v-model="currentPipelineSettings.enableAprilTag"
+      label="Enable AprilTag"
+      :switch-cols="interactiveCols"
+      @update:modelValue="
+        (value) => useCameraSettingsStore().changeCurrentPipelineSetting({ enableAprilTag: value }, false)
+      "
+    />
     <pv-select
       v-model="currentPipelineSettings.tagFamily"
       label="Target family"
       :items="['AprilTag 36h11 (6.5in)', 'AprilTag 16h5 (6in)']"
       :select-cols="interactiveCols"
+      :disabled="aprilTagDisabled"
       @update:modelValue="(value) => useCameraSettingsStore().changeCurrentPipelineSetting({ tagFamily: value }, false)"
     />
     <pv-slider
@@ -36,6 +55,7 @@ const interactiveCols = computed(() =>
       tooltip="Increases FPS at the expense of range by reducing image resolution initially"
       :min="1"
       :max="8"
+      :disabled="aprilTagDisabled"
       @update:modelValue="(value) => useCameraSettingsStore().changeCurrentPipelineSetting({ decimate: value }, false)"
     />
     <pv-slider
@@ -46,6 +66,7 @@ const interactiveCols = computed(() =>
       :min="0"
       :max="5"
       :step="0.1"
+      :disabled="aprilTagDisabled"
       @update:modelValue="(value) => useCameraSettingsStore().changeCurrentPipelineSetting({ blur: value }, false)"
     />
     <pv-slider
@@ -55,6 +76,7 @@ const interactiveCols = computed(() =>
       tooltip="Number of threads spawned by the AprilTag detector"
       :min="1"
       :max="8"
+      :disabled="aprilTagDisabled"
       @update:modelValue="(value) => useCameraSettingsStore().changeCurrentPipelineSetting({ threads: value }, false)"
     />
     <pv-slider
@@ -64,6 +86,7 @@ const interactiveCols = computed(() =>
       tooltip="Tags with a 'margin' (decoding quality score) less than this wil be rejected. Increase this to reduce the number of false positive detections"
       :min="0"
       :max="250"
+      :disabled="aprilTagDisabled"
       @update:modelValue="
         (value) => useCameraSettingsStore().changeCurrentPipelineSetting({ decisionMargin: value }, false)
       "
@@ -75,6 +98,7 @@ const interactiveCols = computed(() =>
       tooltip="Number of iterations the pose estimation algorithm will run, 50-100 is a good starting point"
       :min="0"
       :max="500"
+      :disabled="aprilTagDisabled"
       @update:modelValue="
         (value) => useCameraSettingsStore().changeCurrentPipelineSetting({ numIterations: value }, false)
       "
@@ -84,6 +108,7 @@ const interactiveCols = computed(() =>
       :switch-cols="interactiveCols"
       label="Refine Edges"
       tooltip="Further refines the AprilTag corner position initial estimate, suggested left on"
+      :disabled="aprilTagDisabled"
       @update:modelValue="
         (value) => useCameraSettingsStore().changeCurrentPipelineSetting({ refineEdges: value }, false)
       "

photon-client/src/components/dashboard/tabs/ObjectDetectionTab.vue

@@ -1,9 +1,10 @@
 <script setup lang="ts">
 import { useCameraSettingsStore } from "@/stores/settings/CameraSettingsStore";
-import { type ObjectDetectionPipelineSettings, PipelineType } from "@/types/PipelineTypes";
+import { type CompositePipelineSettings, type ObjectDetectionPipelineSettings, PipelineType } from "@/types/PipelineTypes";
 import PvSlider from "@/components/common/pv-slider.vue";
 import PvSelect from "@/components/common/pv-select.vue";
 import PvRangeSlider from "@/components/common/pv-range-slider.vue";
+import PvSwitch from "@/components/common/pv-switch.vue";
 import { computed } from "vue";
 import { useStateStore } from "@/stores/StateStore";
 import { useSettingsStore } from "@/stores/settings/GeneralSettingsStore";
@@ -12,7 +13,7 @@ import type { ObjectDetectionModelProperties } from "@/types/SettingTypes";
 
 // TODO fix pipeline typing in order to fix this, the store settings call should be able to infer that only valid pipeline type settings are exposed based on pre-checks for the entire config section
 // Defer reference to store access method
-const currentPipelineSettings = computed<ObjectDetectionPipelineSettings>(
+const currentPipelineSettings = computed<ObjectDetectionPipelineSettings | CompositePipelineSettings>(
   () => useCameraSettingsStore().currentPipelineSettings as ObjectDetectionPipelineSettings
 );
 
@@ -31,6 +32,11 @@ const { mdAndDown } = useDisplay();
 const interactiveCols = computed(() =>
   mdAndDown.value && (!useStateStore().sidebarFolded || useCameraSettingsStore().isDriverMode) ? 9 : 8
 );
+const objectDetectionDisabled = computed(
+  () =>
+    currentPipelineSettings.value.pipelineType === PipelineType.Composite &&
+    !currentPipelineSettings.value.enableObjectDetection
+);
 
 // Filters out models that are not supported by the current backend, and returns a flattened list.
 const supportedModels = computed<ObjectDetectionModelProperties[]>(() => {
@@ -63,13 +69,28 @@ const selectedModel = computed({
 </script>
 
 <template>
-  <div v-if="currentPipelineSettings.pipelineType === PipelineType.ObjectDetection">
+  <div
+    v-if="
+      currentPipelineSettings.pipelineType === PipelineType.ObjectDetection ||
+      currentPipelineSettings.pipelineType === PipelineType.Composite
+    "
+  >
+    <pv-switch
+      v-if="currentPipelineSettings.pipelineType === PipelineType.Composite"
+      v-model="currentPipelineSettings.enableObjectDetection"
+      label="Enable Object Detection"
+      :switch-cols="interactiveCols"
+      @update:modelValue="
+        (value) => useCameraSettingsStore().changeCurrentPipelineSetting({ enableObjectDetection: value }, false)
+      "
+    />
     <pv-select
       v-model="selectedModel"
       label="Model"
       tooltip="The model used to detect objects in the camera feed"
       :select-cols="interactiveCols"
       :items="supportedModels.map((model) => model.nickname)"
+      :disabled="objectDetectionDisabled"
     />
 
     <pv-slider
@@ -81,6 +102,7 @@ const selectedModel = computed({
       :min="0"
       :max="1"
       :step="0.01"
+      :disabled="objectDetectionDisabled"
       @update:modelValue="
         (value) => useCameraSettingsStore().changeCurrentPipelineSetting({ confidence: value }, false)
       "
@@ -94,6 +116,7 @@ const selectedModel = computed({
       :min="0"
       :max="1"
       :step="0.01"
+      :disabled="objectDetectionDisabled"
       @update:modelValue="(value) => useCameraSettingsStore().changeCurrentPipelineSetting({ nms: value }, false)"
     />
     <pv-range-slider
@@ -103,6 +126,7 @@ const selectedModel = computed({
       :max="100"
       :slider-cols="interactiveCols"
       :step="0.01"
+      :disabled="objectDetectionDisabled"
       @update:modelValue="
         (value) => useCameraSettingsStore().changeCurrentPipelineSetting({ contourArea: value }, false)
       "
@@ -115,6 +139,7 @@ const selectedModel = computed({
       :max="100"
       :slider-cols="interactiveCols"
       :step="0.01"
+      :disabled="objectDetectionDisabled"
       @update:modelValue="
         (value) => useCameraSettingsStore().changeCurrentPipelineSetting({ contourRatio: value }, false)
       "
@@ -125,6 +150,7 @@ const selectedModel = computed({
       tooltip="Used to determine how to calculate target landmarks, as well as aspect ratio"
       :items="['Portrait', 'Landscape']"
       :select-cols="interactiveCols"
+      :disabled="objectDetectionDisabled"
       @update:modelValue="
         (value) =>
           useCameraSettingsStore().changeCurrentPipelineSetting(
@@ -139,6 +165,7 @@ const selectedModel = computed({
       tooltip="Chooses the sorting mode used to determine the 'best' targets to provide to user code"
       :select-cols="interactiveCols"
       :items="['Largest', 'Smallest', 'Highest', 'Lowest', 'Rightmost', 'Leftmost', 'Centermost']"
+      :disabled="objectDetectionDisabled"
       @update:modelValue="
         (value) =>
           useCameraSettingsStore().changeCurrentPipelineSetting(

photon-client/src/components/dashboard/tabs/OutputTab.vue

@@ -18,6 +18,10 @@ const isTagPipeline = computed(
     useCameraSettingsStore().currentPipelineType === PipelineType.Aruco
 );
 
+const isFiducialPipeline = computed(
+  () => isTagPipeline.value || useCameraSettingsStore().currentPipelineType === PipelineType.Composite
+);
+
 interface MetricItem {
   header: string;
   value?: string;
@@ -75,31 +79,40 @@ const interactiveCols = computed(() =>
     <pv-switch
       v-if="
         (currentPipelineSettings.pipelineType === PipelineType.AprilTag ||
-          currentPipelineSettings.pipelineType === PipelineType.Aruco) &&
+          currentPipelineSettings.pipelineType === PipelineType.Aruco ||
+          currentPipelineSettings.pipelineType === PipelineType.Composite) &&
         useCameraSettingsStore().isCurrentVideoFormatCalibrated &&
         useCameraSettingsStore().currentPipelineSettings.solvePNPEnabled
       "
       v-model="currentPipelineSettings.doMultiTarget"
       label="Do Multi-Target Estimation"
       tooltip="If enabled, all visible fiducial targets will be used to provide a single pose estimate from their combined model."
       :switch-cols="interactiveCols"
-      :disabled="!isTagPipeline"
+      :disabled="
+        !isFiducialPipeline ||
+        (currentPipelineSettings.pipelineType === PipelineType.Composite && !currentPipelineSettings.enableAprilTag)
+      "
       @update:modelValue="
         (value) => useCameraSettingsStore().changeCurrentPipelineSetting({ doMultiTarget: value }, false)
       "
     />
     <pv-switch
       v-if="
         (currentPipelineSettings.pipelineType === PipelineType.AprilTag ||
-          currentPipelineSettings.pipelineType === PipelineType.Aruco) &&
+          currentPipelineSettings.pipelineType === PipelineType.Aruco ||
+          currentPipelineSettings.pipelineType === PipelineType.Composite) &&
         useCameraSettingsStore().isCurrentVideoFormatCalibrated &&
         useCameraSettingsStore().currentPipelineSettings.solvePNPEnabled
       "
       v-model="currentPipelineSettings.doSingleTargetAlways"
       label="Always Do Single-Target Estimation"
       tooltip="If disabled, visible fiducial targets used for multi-target estimation will not also be used for single-target estimation."
       :switch-cols="interactiveCols"
-      :disabled="!isTagPipeline || !currentPipelineSettings.doMultiTarget"
+      :disabled="
+        !isFiducialPipeline ||
+        !currentPipelineSettings.doMultiTarget ||
+        (currentPipelineSettings.pipelineType === PipelineType.Composite && !currentPipelineSettings.enableAprilTag)
+      "
       @update:modelValue="
         (value) => useCameraSettingsStore().changeCurrentPipelineSetting({ doSingleTargetAlways: value }, false)
       "