Recording Playback (#7792)

* recordings can be searched for per user

* can deserialize recording entity data

* playwork works

* WIP capture route and UI component

* playback of data channel data works

* update

* split up concepts of avatar uuid and $from actions, playback of ecs state sort of works

* refactor IK targets

* properly destroy entity after recording finished

* add twin UI

* fix tests

---------

Co-authored-by: jerknose <[email protected]>

Author: HexaField

package.json

@@ -145,6 +145,7 @@
     "@feathersjs/feathers": "5.0.0",
     "app-root-path": "3.0.0",
     "cli": "1.0.1",
+    "daisyui": "^2.51.4",
     "dotenv": "14.3.2",
     "fix-esm": "^1.0.1",
     "lerna": "4.0.0",
@@ -154,6 +155,7 @@
     "postcss": "^8.4.16",
     "qrcode.react": "^3.1.0",
     "react-json-view": "^1.21.3",
+    "tailwindcss": "^3.2.7",
     "three": "0.150.1",
     "ts-node": "10.8.2",
     "typescript": "4.9.5",

packages/client-core/src/components/Debug/styles.module.scss

@@ -80,7 +80,7 @@
 
 .refreshBlock {
   display: flex;
-  justify-content: end;
+  justify-content: flex-end;
   gap: 10px;
 
   .refreshBtn {

packages/client-core/src/components/FullscreenContainer.tsx

@@ -38,8 +38,8 @@ export const FullscreenContainer = React.forwardRef((props: Props, ref: any) =>
     </div>
   ) : (
     <FullscreenContext.Provider value={[fullScreenActive.value, fullScreenActive.set]}>
-      <FullScreen handle={handle} onChange={reportChange} className="h-full">
-        <div id={'engine-container'} ref={ref} className="h-full">
+      <FullScreen handle={handle} onChange={reportChange}>
+        <div id={'engine-container'} ref={ref}>
           {props.children}
         </div>
       </FullScreen>

packages/client-core/src/components/InstanceChat/index.module.scss

@@ -42,7 +42,7 @@
     display: flex;
     flex-direction: row-reverse;
     align-items: center;
-    justify-content: end;
+    justify-content: flex-end;
     transition: all 0.2s;
 
     .chat-input {

packages/client-core/src/components/LocationIcons/index.module.scss

@@ -1,7 +1,7 @@
 .rightSidebar {
   display: flex;
   flex-direction: column;
-  align-items: end;
+  align-items: flex-end;
   justify-content: space-between;
   position: fixed;
   top: 0;

packages/client-core/src/components/MediaIconsBox/index.tsx

@@ -102,7 +102,7 @@ export const MediaIconsBox = () => {
             type="button"
             id="UserPoseTracking"
             className={styles.iconContainer + ' ' + (isMotionCaptureEnabled.value ? styles.on : '')}
-            onClick={() => window.open('/recorder', '_blank')}
+            onClick={() => window.open('/capture', '_blank')}
             onPointerUp={() => AudioEffectPlayer.instance.play(AudioEffectPlayer.SOUNDS.ui)}
             onPointerEnter={() => AudioEffectPlayer.instance.play(AudioEffectPlayer.SOUNDS.ui)}
           >

packages/client-core/src/components/UserMediaWindows/index.module.scss

@@ -17,7 +17,7 @@
 .userMediaWindows {
   display: flex;
   flex-direction: column;
-  align-items: end;
+  align-items: flex-end;
   padding: 0 15px;
   overflow-y: auto;
   scrollbar-width: none;
@@ -28,7 +28,7 @@
     margin-top: 5px;
     flex-direction: row;
     align-items: center;
-    justify-content: end;
+    justify-content: flex-end;
 
     .conferenceModeButton {
       padding: 1px;

packages/client-core/src/media/webcam/WebcamInput.ts

@@ -15,6 +15,7 @@ import {
 import { WebcamInputComponent } from '@etherealengine/engine/src/input/components/WebcamInputComponent'
 import { WorldNetworkAction } from '@etherealengine/engine/src/networking/functions/WorldNetworkAction'
 import { GroupComponent } from '@etherealengine/engine/src/scene/components/GroupComponent'
+import { UUIDComponent } from '@etherealengine/engine/src/scene/components/UUIDComponent'
 import { createActionQueue, getMutableState } from '@etherealengine/hyperflux'
 
 import { MediaStreamState } from '../../transports/MediaStreams'
@@ -259,7 +260,7 @@ export default async function WebcamInputSystem() {
 
   const execute = () => {
     for (const action of avatarSpawnQueue()) {
-      const entity = Engine.instance.getUserAvatarEntity(action.$from)
+      const entity = UUIDComponent.entitiesByUUID.value[action.uuid]
       setComponent(entity, WebcamInputComponent)
     }
     for (const entity of webcamQuery()) setAvatarExpression(entity)

packages/client-core/src/recording/RecordingService.ts

@@ -18,7 +18,9 @@ export const RecordingState = defineState({
       mocap: true,
       video: true,
       pose: true
-    }
+    },
+    recordings: [] as RecordingResult[],
+    playback: null as string | null
   }
 })
 
@@ -46,6 +48,11 @@ export const RecordingFunctions = {
     } catch (err) {
       NotificationService.dispatchNotify(err.message, { variant: 'error' })
     }
+  },
+  getRecordings: async () => {
+    const recordings = (await API.instance.client.service('recording').find()).data as RecordingResult[]
+    const recordingState = getMutableState(RecordingState)
+    recordingState.recordings.set(recordings)
   }
 }
 
@@ -55,6 +62,7 @@ export async function RecordingStateReceptorSystem() {
   const startRecordingQueue = createActionQueue(ECSRecordingActions.startRecording.matches)
   const recordingStartedQueue = createActionQueue(ECSRecordingActions.recordingStarted.matches)
   const stopRecordingQueue = createActionQueue(ECSRecordingActions.stopRecording.matches)
+  const playbackChangedQueue = createActionQueue(ECSRecordingActions.playbackChanged.matches)
 
   const execute = () => {
     for (const action of recordingStartedQueue()) {
@@ -70,10 +78,17 @@ export async function RecordingStateReceptorSystem() {
       recordingState.started.set(false)
       recordingState.recordingID.set(null)
     }
+
+    for (const action of playbackChangedQueue()) {
+      recordingState.playback.set(action.playing ? action.recordingID : null)
+    }
   }
 
   const cleanup = async () => {
+    removeActionQueue(startRecordingQueue)
     removeActionQueue(recordingStartedQueue)
+    removeActionQueue(stopRecordingQueue)
+    removeActionQueue(playbackChangedQueue)
   }
 
   return { execute, cleanup }

packages/client-core/src/systems/ui/SettingDetailView/index.scss

@@ -52,7 +52,7 @@
 .controlsContainer {
   display: flex;
   justify-content: space-between;
-  align-items: end;
+  align-items: flex-end;
 }
 
 .selectSize {