Refactor Network and Media State (#7887)

* remove multiple sources of truth for media consumers, remove redundant actions

* remove duplicated state and actions for media

* more cleanup

Author: HexaField

packages/client-core/src/common/services/LocationInstanceConnectionService.ts

@@ -1,5 +1,5 @@
 import { Paginated } from '@feathersjs/feathers'
-import { none } from '@hookstate/core'
+import { none, State } from '@hookstate/core'
 import { useEffect } from 'react'
 
 import { Instance } from '@etherealengine/common/src/interfaces/Instance'
@@ -53,6 +53,12 @@ export const LocationInstanceState = defineState({
   })
 })
 
+export function useWorldNetwork() {
+  const worldNetworkState = useState(getMutableState(NetworkState).networks)
+  const worldHostId = useState(getMutableState(NetworkState).hostIds.world)
+  return worldHostId.value ? (worldNetworkState[worldHostId.value] as State<SocketWebRTCClientNetwork>) : null
+}
+
 export function useWorldInstance() {
   const worldInstanceState = useState(getMutableState(LocationInstanceState).instances)
   const worldHostId = useState(getMutableState(NetworkState).hostIds.world)

packages/client-core/src/common/services/MediaInstanceConnectionService.ts

@@ -53,6 +53,12 @@ export const MediaInstanceState = defineState({
   })
 })
 
+export function useMediaNetwork() {
+  const mediaNetworkState = useState(getMutableState(NetworkState).networks)
+  const mediaHostId = useState(getMutableState(NetworkState).hostIds.media)
+  return mediaHostId.value ? (mediaNetworkState[mediaHostId.value] as State<SocketWebRTCClientNetwork>) : null
+}
+
 export function useMediaInstance() {
   const mediaInstanceState = useState(getMutableState(MediaInstanceState).instances)
   const mediaHostId = useState(getMutableState(NetworkState).hostIds.media)

packages/client-core/src/components/ConferenceMode/index.tsx

@@ -3,19 +3,19 @@ import classNames from 'classnames'
 import React from 'react'
 
 import { MediaInstanceState } from '@etherealengine/client-core/src/common/services/MediaInstanceConnectionService'
-import { MediaState } from '@etherealengine/client-core/src/media/services/MediaStreamService'
 import { AuthState } from '@etherealengine/client-core/src/user/services/AuthService'
 import { NetworkUserState } from '@etherealengine/client-core/src/user/services/NetworkUserService'
 import { Engine } from '@etherealengine/engine/src/ecs/classes/Engine'
 import { screenshareVideoDataChannelType } from '@etherealengine/engine/src/networking/NetworkState'
 import { getMutableState, useHookstate } from '@etherealengine/hyperflux'
 
+import { MediaStreamState } from '../../transports/MediaStreams'
+import { NearbyUsersState } from '../../transports/UpdateNearbyUsersSystem'
 import ConferenceModeParticipant from './ConferenceModeParticipant'
 import styles from './index.module.scss'
 
 const ConferenceMode = (): JSX.Element => {
-  const mediaState = useHookstate(getMutableState(MediaState))
-  const nearbyLayerUsers = mediaState.nearbyLayerUsers
+  const nearbyLayerUsers = useHookstate(getMutableState(NearbyUsersState).nearbyLayerUsers)
   const selfUserId = useHookstate(getMutableState(AuthState).user.id)
   const userState = useHookstate(getMutableState(NetworkUserState))
   const channelConnectionState = useHookstate(getMutableState(MediaInstanceState))
@@ -36,13 +36,19 @@ const ConferenceMode = (): JSX.Element => {
         : []
       : []
 
-  const consumers = mediaState.consumers.value
+  const consumers = network.consumers
   const screenShareConsumers =
     consumers?.filter((consumer) => consumer.appData.mediaTag === screenshareVideoDataChannelType) || []
 
+  const mediaStreamState = useHookstate(getMutableState(MediaStreamState))
+  const isScreenVideoEnabled =
+    mediaStreamState.screenVideoProducer.value != null && !mediaStreamState.screenShareVideoPaused.value
+  const isScreenAudioEnabled =
+    mediaStreamState.screenShareAudioPaused.value != null && !mediaStreamState.screenShareAudioPaused.value
+
   let totalScreens = 1
 
-  if (mediaState.isScreenAudioEnabled.value || mediaState.isScreenVideoEnabled.value) {
+  if (isScreenVideoEnabled || isScreenAudioEnabled) {
     totalScreens += 1
   }
 
@@ -63,7 +69,7 @@ const ConferenceMode = (): JSX.Element => {
         [styles['multi-grid']]: totalScreens === 3 || totalScreens > 4
       })}
     >
-      {(mediaState.isScreenAudioEnabled.value || mediaState.isScreenVideoEnabled.value) && (
+      {(isScreenVideoEnabled || isScreenAudioEnabled) && (
         <ConferenceModeParticipant type={'screen'} peerID={network.peerID} key={'screen_' + network.peerID} />
       )}
       <ConferenceModeParticipant type={'cam'} peerID={network.peerID} key={'cam_' + network.peerID} />

packages/client-core/src/components/MediaIconsBox/index.tsx

@@ -1,11 +1,8 @@
 import React, { useEffect, useState } from 'react'
-import { Navigate, useNavigate } from 'react-router-dom'
 
 import { useMediaInstanceConnectionState } from '@etherealengine/client-core/src/common/services/MediaInstanceConnectionService'
-import { useMediaStreamState } from '@etherealengine/client-core/src/media/services/MediaStreamService'
 import { useLocationState } from '@etherealengine/client-core/src/social/services/LocationService'
 import {
-  toggleFaceTracking,
   toggleMicrophonePaused,
   toggleScreenshare,
   toggleWebcamPaused
@@ -19,11 +16,11 @@ import { dispatchAction, getMutableState, useHookstate } from '@etherealengine/h
 import Icon from '@etherealengine/ui/src/Icon'
 
 import { VrIcon } from '../../common/components/Icons/VrIcon'
+import { MediaStreamState } from '../../transports/MediaStreams'
 import { useShelfStyles } from '../Shelves/useShelfStyles'
 import styles from './index.module.scss'
 
 export const MediaIconsBox = () => {
-  const navigate = useNavigate()
   const [hasAudioDevice, setHasAudioDevice] = useState(false)
   const [hasVideoDevice, setHasVideoDevice] = useState(false)
   const { topShelfStyle } = useShelfStyles()
@@ -32,18 +29,19 @@ export const MediaIconsBox = () => {
   const channelConnectionState = useMediaInstanceConnectionState()
   const mediaHostId = Engine.instance.mediaNetwork?.hostId
   const currentChannelInstanceConnection = mediaHostId && channelConnectionState.instances[mediaHostId].ornull
-  const mediastream = useMediaStreamState()
   const videoEnabled = currentLocation?.locationSetting?.value
     ? currentLocation?.locationSetting?.videoEnabled?.value
     : false
   const audioEnabled = currentLocation?.locationSetting?.value
     ? currentLocation?.locationSetting?.audioEnabled?.value
     : false
 
-  const isMotionCaptureEnabled = mediastream.isMotionCaptureEnabled
-  const isCamVideoEnabled = mediastream.isCamVideoEnabled
-  const isCamAudioEnabled = mediastream.isCamAudioEnabled
-  const isScreenVideoEnabled = mediastream.isScreenVideoEnabled
+  const mediaStreamState = useHookstate(getMutableState(MediaStreamState))
+  const isMotionCaptureEnabled = mediaStreamState.faceTracking.value
+  const isCamVideoEnabled = mediaStreamState.camVideoProducer.value != null && !mediaStreamState.videoPaused.value
+  const isCamAudioEnabled = mediaStreamState.camAudioProducer.value != null && !mediaStreamState.audioPaused.value
+  const isScreenVideoEnabled =
+    mediaStreamState.screenVideoProducer.value != null && !mediaStreamState.screenShareVideoPaused.value
 
   const engineState = useEngineState()
   const xrState = useHookstate(getMutableState(XRState))
@@ -75,12 +73,12 @@ export const MediaIconsBox = () => {
         <button
           type="button"
           id="UserAudio"
-          className={styles.iconContainer + ' ' + (isCamAudioEnabled.value ? styles.on : '')}
+          className={styles.iconContainer + ' ' + (isCamAudioEnabled ? styles.on : '')}
           onClick={toggleMicrophonePaused}
           onPointerUp={() => AudioEffectPlayer.instance.play(AudioEffectPlayer.SOUNDS.ui)}
           onPointerEnter={() => AudioEffectPlayer.instance.play(AudioEffectPlayer.SOUNDS.ui)}
         >
-          <Icon type={isCamAudioEnabled.value ? 'Mic' : 'MicOff'} />
+          <Icon type={isCamAudioEnabled ? 'Mic' : 'MicOff'} />
         </button>
       ) : null}
       {videoEnabled &&
@@ -91,17 +89,17 @@ export const MediaIconsBox = () => {
           <button
             type="button"
             id="UserVideo"
-            className={styles.iconContainer + ' ' + (isCamVideoEnabled.value ? styles.on : '')}
+            className={styles.iconContainer + ' ' + (isCamVideoEnabled ? styles.on : '')}
             onClick={toggleWebcamPaused}
             onPointerUp={() => AudioEffectPlayer.instance.play(AudioEffectPlayer.SOUNDS.ui)}
             onPointerEnter={() => AudioEffectPlayer.instance.play(AudioEffectPlayer.SOUNDS.ui)}
           >
-            <Icon type={isCamVideoEnabled.value ? 'Videocam' : 'VideocamOff'} />
+            <Icon type={isCamVideoEnabled ? 'Videocam' : 'VideocamOff'} />
           </button>
           <button
             type="button"
             id="UserPoseTracking"
-            className={styles.iconContainer + ' ' + (isMotionCaptureEnabled.value ? styles.on : '')}
+            className={styles.iconContainer + ' ' + (isMotionCaptureEnabled ? styles.on : '')}
             onClick={() => window.open('/capture', '_blank')}
             onPointerUp={() => AudioEffectPlayer.instance.play(AudioEffectPlayer.SOUNDS.ui)}
             onPointerEnter={() => AudioEffectPlayer.instance.play(AudioEffectPlayer.SOUNDS.ui)}
@@ -111,7 +109,7 @@ export const MediaIconsBox = () => {
           <button
             type="button"
             id="UserScreenSharing"
-            className={styles.iconContainer + ' ' + (isScreenVideoEnabled.value ? styles.on : '')}
+            className={styles.iconContainer + ' ' + (isScreenVideoEnabled ? styles.on : '')}
             onClick={toggleScreenshare}
             onPointerUp={() => AudioEffectPlayer.instance.play(AudioEffectPlayer.SOUNDS.ui)}
             onPointerEnter={() => AudioEffectPlayer.instance.play(AudioEffectPlayer.SOUNDS.ui)}

packages/client-core/src/components/UserMediaWindow/index.tsx

@@ -4,7 +4,6 @@ import hark from 'hark'
 import React, { useEffect, useRef, useState } from 'react'
 import { useTranslation } from 'react-i18next'
 
-import { useMediaStreamState } from '@etherealengine/client-core/src/media/services/MediaStreamService'
 import { useLocationState } from '@etherealengine/client-core/src/social/services/LocationService'
 import {
   globalMuteProducer,

packages/client-core/src/media/PeerMedia.tsx

@@ -5,14 +5,19 @@ import { Engine } from '@etherealengine/engine/src/ecs/classes/Engine'
 import { MessageTypes } from '@etherealengine/engine/src/networking/enums/MessageTypes'
 import {
   MediaTagType,
+  NetworkState,
   screenshareAudioDataChannelType,
   screenshareVideoDataChannelType,
   webcamAudioDataChannelType,
   webcamVideoDataChannelType
 } from '@etherealengine/engine/src/networking/NetworkState'
-import { getMutableState, useHookstate } from '@etherealengine/hyperflux'
+import { getMutableState, State, useHookstate } from '@etherealengine/hyperflux'
 
-import { MediaInstanceState } from '../common/services/MediaInstanceConnectionService'
+import {
+  MediaInstanceState,
+  useMediaInstance,
+  useMediaNetwork
+} from '../common/services/MediaInstanceConnectionService'
 import { MediaStreamState } from '../transports/MediaStreams'
 import {
   createPeerMediaChannels,
@@ -24,13 +29,14 @@ import {
   ProducerExtension,
   SocketWebRTCClientNetwork
 } from '../transports/SocketWebRTCClientFunctions'
+import { NearbyUsersState } from '../transports/UpdateNearbyUsersSystem'
 import { AuthState } from '../user/services/AuthService'
 import { NetworkUserState } from '../user/services/NetworkUserService'
-import { MediaState } from './services/MediaStreamService'
 
 export const getMediaChannels = (network: SocketWebRTCClientNetwork, consumers: ConsumerExtension[]) => {
-  const mediaState = getMutableState(MediaState)
-  const nearbyLayerUsers = mediaState.nearbyLayerUsers
+  const nearbyUsersState = getMutableState(NearbyUsersState)
+  const mediaStreamState = getMutableState(MediaStreamState)
+  const nearbyLayerUsers = nearbyUsersState.nearbyLayerUsers
   const selfUserId = getMutableState(AuthState).user.id
   const userState = getMutableState(NetworkUserState)
   const channelConnectionState = getMutableState(MediaInstanceState)
@@ -56,9 +62,9 @@ export const getMediaChannels = (network: SocketWebRTCClientNetwork, consumers:
 
   /** always put own peer first */
   const selfPeerID = network?.peerID ?? 'self'
-  if (mediaState.isScreenVideoEnabled.value)
+  if (mediaStreamState.screenVideoProducer.value != null && !mediaStreamState.screenShareVideoPaused.value)
     mediaChannels.push({ peerID: selfPeerID, mediaTag: screenshareVideoDataChannelType })
-  if (mediaState.isScreenAudioEnabled.value)
+  if (mediaStreamState.screenAudioProducer.value != null && !mediaStreamState.screenShareAudioPaused.value)
     mediaChannels.push({ peerID: selfPeerID, mediaTag: screenshareAudioDataChannelType })
   mediaChannels.push({ peerID: selfPeerID, mediaTag: webcamVideoDataChannelType })
   mediaChannels.push({ peerID: selfPeerID, mediaTag: webcamAudioDataChannelType })
@@ -88,7 +94,7 @@ export const getMediaChannels = (network: SocketWebRTCClientNetwork, consumers:
 /**
  * Sets media stream state for a peer
  */
-const PeerConsumer = (props: {
+const PeerMedia = (props: {
   channel: ConsumerExtension | ProducerExtension
   peerID: PeerID
   mediaTag: MediaTagType
@@ -128,13 +134,13 @@ const PeerConsumer = (props: {
       peerMediaChannelState.audioProducerPaused.set(true)
       peerMediaChannelState.audioProducerGlobalMute.set(true)
     } else {
-      const videoConsumer = network.consumers?.find(
+      const videoConsumer = network.consumers.find(
         (c) =>
           c.appData.peerID === peerID &&
           c.producerId === producerId &&
           c.appData.mediaTag === (isScreen ? screenshareVideoDataChannelType : webcamVideoDataChannelType)
       )
-      const audioConsumer = network.consumers?.find(
+      const audioConsumer = network.consumers.find(
         (c) =>
           c.appData.peerID === peerID &&
           c.producerId === producerId &&
@@ -159,12 +165,12 @@ const PeerConsumer = (props: {
       peerMediaChannelState.audioProducerPaused.set(false)
       peerMediaChannelState.audioProducerGlobalMute.set(false)
     } else {
-      const videoConsumer = network.consumers?.find(
+      const videoConsumer = network.consumers.find(
         (c) =>
           c.appData.peerID === peerID &&
           c.appData.mediaTag === (isScreen ? screenshareVideoDataChannelType : webcamVideoDataChannelType)
       )
-      const audioConsumer = network.consumers?.find(
+      const audioConsumer = network.consumers.find(
         (c) =>
           c.appData.peerID === peerID &&
           c.appData.mediaTag === (isScreen ? screenshareAudioDataChannelType : webcamAudioDataChannelType)
@@ -213,12 +219,12 @@ const PeerConsumer = (props: {
         }
       }
     } else {
-      const videoConsumer = network.consumers?.find(
+      const videoConsumer = network.consumers.find(
         (c) =>
           c.appData.peerID === peerID &&
           c.appData.mediaTag === (isScreen ? screenshareVideoDataChannelType : webcamVideoDataChannelType)
       )
-      const audioConsumer = network.consumers?.find(
+      const audioConsumer = network.consumers.find(
         (c) =>
           c.appData.peerID === peerID &&
           c.appData.mediaTag === (isScreen ? screenshareAudioDataChannelType : webcamAudioDataChannelType)
@@ -304,15 +310,19 @@ const PeerConsumer = (props: {
   return null
 }
 
-export const PeerMedia = () => {
-  const mediaState = useHookstate(getMutableState(MediaState))
+export const PeerConsumers = () => {
+  const nearbyUsersState = useHookstate(getMutableState(NearbyUsersState))
+
   const mediaStreamState = useHookstate(getMutableState(MediaStreamState))
   const peerMediaChannelState = useHookstate(getMutableState(PeerMediaChannelState))
-  const network = Engine.instance.mediaNetwork as SocketWebRTCClientNetwork
+
+  const networkState = useMediaNetwork()
+  const network = networkState?.value
 
   // create a peer media stream for each peer with a consumer
   useEffect(() => {
-    const mediaChannels = getMediaChannels(network, mediaState.consumers.get({ noproxy: true }))
+    if (!network) return
+    const mediaChannels = getMediaChannels(network, network.consumers)
     for (const consumer of mediaChannels) {
       if (!peerMediaChannelState.value[consumer.peerID]) {
         createPeerMediaChannels(consumer.peerID)
@@ -325,8 +335,8 @@ export const PeerMedia = () => {
       }
     }
   }, [
-    mediaState.nearbyLayerUsers.length,
-    mediaState.consumers.length,
+    nearbyUsersState.nearbyLayerUsers.length,
+    networkState?.consumers?.length,
     mediaStreamState.videoStream,
     mediaStreamState.audioStream,
     mediaStreamState.screenAudioProducer,
@@ -339,14 +349,15 @@ export const PeerMedia = () => {
     channel: ConsumerExtension | ProducerExtension
   }[]
 
-  mediaChannels.push(
-    ...mediaState.consumers.value.map((media) => {
-      return { peerID: media.appData.peerID, mediaTag: media.appData.mediaTag, channel: media }
-    })
-  )
+  if (network)
+    mediaChannels.push(
+      ...network.consumers.map((media: ConsumerExtension) => {
+        return { peerID: media.appData.peerID, mediaTag: media.appData.mediaTag, channel: media }
+      })
+    )
 
   // own peer id
-  const peerID = network?.peerID ?? 'self'
+  const peerID = network?.peerID ?? ('self' as PeerID)
 
   if (mediaStreamState.camVideoProducer.value)
     mediaChannels.push({
@@ -378,7 +389,7 @@ export const PeerMedia = () => {
       {mediaChannels
         .filter(({ peerID }) => peerMediaChannelState[peerID].value)
         .map(({ channel, peerID, mediaTag }) => (
-          <PeerConsumer channel={channel} peerID={peerID} mediaTag={mediaTag} key={peerID + '-' + mediaTag} />
+          <PeerMedia channel={channel} peerID={peerID} mediaTag={mediaTag} key={peerID + '-' + mediaTag} />
         ))}
     </>
   )