Updated client to prefer lowest-res video when video is small.

packages/client-core/src/components/UserMediaWindow/index.tsx

@@ -34,6 +34,7 @@ import {
   globalUnmuteProducer,
   pauseConsumer,
   resumeConsumer,
+  setPreferredConsumerLayer,
   toggleMicrophonePaused,
   toggleScreenshareAudioPaused,
   toggleScreenshareVideoPaused,
@@ -434,6 +435,35 @@ export const UserMediaWindow = ({ peerID, type }: Props): JSX.Element => {
     document.getElementById(peerID + '-' + type + '-audio-container')!.append(audioElement)
   }, [])
 
+  useEffect(() => {
+    if (!videoStream) return
+    const mediaNetwork = Engine.instance.mediaNetwork as SocketWebRTCClientNetwork
+    const encodings = videoStream.rtpParameters.encodings
+
+    const immersiveMedia = getMutableState(MediaSettingsState).immersiveMedia
+    if (isPiP || immersiveMedia.value) {
+      let maxLayer
+      const scalabilityMode = encodings && encodings[0].scalabilityMode
+      if (!scalabilityMode) maxLayer = 0
+      else {
+        const execed = /L([0-9])/.exec(scalabilityMode)
+        if (execed) maxLayer = parseInt(execed[1]) - 1 //Subtract 1 from max scalabilityMode since layers are 0-indexed
+        else maxLayer = 0
+      }
+      // If we're in immersive media mode, using max-resolution video for everyone could overwhelm some devices.
+      // If there are more than 2 layers, then use layer n-1 to balance quality and performance
+      // (immersive video bubbles are bigger than the flat bubbles, so low-quality video will be more noticeable).
+      // If we're not, then use the highest layer when opening PiP for a video
+      setPreferredConsumerLayer(
+        mediaNetwork,
+        videoStream as ConsumerExtension,
+        immersiveMedia && maxLayer > 1 ? maxLayer - 1 : maxLayer
+      )
+    }
+    // Standard video bubbles in flat/non-immersive mode should use the lowest quality layer for performance reasons
+    else setPreferredConsumerLayer(mediaNetwork, videoStream as ConsumerExtension, 0)
+  }, [videoStream, isPiP])
+
   return (
     <Draggable isPiP={isPiP}>
       <div

packages/client-core/src/components/UserMediaWindows/index.tsx

@@ -33,6 +33,7 @@ import { getMutableState } from '@etherealengine/hyperflux'
 
 import { useMediaInstance } from '../../common/services/MediaInstanceConnectionService'
 import { PeerMediaChannelState, PeerMediaStreamInterface } from '../../transports/PeerMediaChannelState'
+import { AuthState } from '../../user/services/AuthService'
 import { useShelfStyles } from '../Shelves/useShelfStyles'
 import { UserMediaWindow, UserMediaWindowWidget } from '../UserMediaWindow'
 import styles from './index.module.scss'
@@ -49,6 +50,7 @@ export const UserMediaWindows = () => {
   const peerMediaChannelState = useHookstate(getMutableState(PeerMediaChannelState))
   const mediaNetworkInstanceState = useMediaInstance()
   const mediaNetwork = Engine.instance.mediaNetwork
+  const selfUser = useHookstate(getMutableState(AuthState).user)
   const mediaNetworkConnected = mediaNetwork && mediaNetworkInstanceState?.connected?.value
 
   const consumers = Object.entries(peerMediaChannelState.get({ noproxy: true })) as [
@@ -70,6 +72,7 @@ export const UserMediaWindows = () => {
   // reduce all userPeers to an array 'windows' of { peerID, type } objects, displaying screens first, then cams. if a user has no cameras, only include one peerID for that user
   const windows = userPeers
     .reduce((acc, [userID, peerIDs]) => {
+      const isSelfWindows = userID === selfUser.id.value
       const userCams = consumers
         .filter(([peerID, { cam, screen }]) => peerIDs.includes(peerID) && cam && camActive(cam))
         .map(([peerID]) => {
@@ -84,9 +87,11 @@ export const UserMediaWindows = () => {
 
       const userWindows = [...userScreens, ...userCams]
       if (userWindows.length) {
-        acc.push(...userWindows)
+        if (isSelfWindows) acc.unshift(...userWindows)
+        else acc.push(...userWindows)
       } else {
-        acc.push({ peerID: peerIDs[0], type: 'cam' })
+        if (isSelfWindows) acc.unshift({ peerID: peerIDs[0], type: 'cam' })
+        else acc.push({ peerID: peerIDs[0], type: 'cam' })
       }
       return acc
     }, [] as WindowType[])

packages/client-core/src/transports/SocketWebRTCClientFunctions.ts

@@ -1370,6 +1370,17 @@ export async function closeConsumer(network: SocketWebRTCClientNetwork, consumer
   })
 }
 
+export async function setPreferredConsumerLayer(
+  network: SocketWebRTCClientNetwork,
+  consumer: ConsumerExtension,
+  layer: number
+) {
+  await promisedRequest(network, MessageTypes.WebRTCConsumerSetLayers.toString(), {
+    consumerId: consumer.id,
+    spatialLayer: layer
+  })
+}
+
 const checkEndVideoChat = async () => {
   const mediaStreamState = getMutableState(MediaStreamState)
   const mediaNetwork = Engine.instance.mediaNetwork as SocketWebRTCClientNetwork

packages/client-core/src/user/components/UserMenu/index.module.scss

@@ -13,7 +13,7 @@
   flex-wrap: wrap;
   z-index: 5;
   display: flex;
-  align-content: end;
+  align-content: flex-end;
   align-items: self-start;
 
   .buttonsContainer {

packages/instanceserver/package.json

@@ -54,14 +54,14 @@
     "@google-cloud/agones-sdk": "1.30.0",
     "@swc/core": "^1.3.41",
     "cross-env": "7.0.3",
+    "detect-port": "^1.5.1",
     "ffmpeg-static": "^5.1.0",
     "jsonwebtoken": "^9.0.0",
     "lodash": "^4.17.21",
     "mediasoup": "3.11.14",
     "msgpackr": "^1.9.2",
     "primus": "^8.0.7",
     "ps-list": "7.2.0",
-    "tcp-port-used": "^1.0.2",
     "trace-unhandled": "2.0.1",
     "ts-node": "10.9.1",
     "typescript": "5.0.2",

packages/instanceserver/src/WebRTCFunctions.ts

@@ -23,6 +23,7 @@ All portions of the code written by the Ethereal Engine team are Copyright © 20
 Ethereal Engine. All Rights Reserved.
 */
 
+import detect from 'detect-port'
 import { createWorker } from 'mediasoup'
 import {
   DataConsumer,
@@ -41,7 +42,6 @@ import {
 } from 'mediasoup/node/lib/types'
 import os from 'os'
 import { Spark } from 'primus'
-import { check } from 'tcp-port-used'
 
 import { PeerID } from '@etherealengine/common/src/interfaces/PeerID'
 import { DataChannelType } from '@etherealengine/engine/src/networking/classes/Network'
@@ -72,8 +72,9 @@ const logger = multiLogger.child({ component: 'instanceserver:webrtc' })
 const portsInUse: number[] = []
 
 const getNewOffset = async (ipAddress, startPort, i, offset) => {
-  const inUse = await check(startPort + i + offset, ipAddress)
-  if (inUse || portsInUse.indexOf(startPort + i + offset) >= 0) return getNewOffset(ipAddress, startPort, i, offset + 1)
+  const inUse = await detect(startPort + i + offset, ipAddress)
+  if (inUse !== startPort + i + offset || portsInUse.indexOf(startPort + i + offset) >= 0)
+    return getNewOffset(ipAddress, startPort, i, offset + 1)
   else return offset
 }
 
@@ -540,7 +541,6 @@ export async function handleWebRtcProduceData(
   messageId?: string
 ): Promise<any> {
   try {
-    console.log('webRTCProduceData')
     const userId = getUserIdFromPeerID(network, peerID)
     if (!userId) {
       logger.info('userId could not be found for sparkID ' + peerID)
@@ -992,9 +992,24 @@ export async function handleWebRtcConsumerSetLayers(
 ): Promise<any> {
   const { consumerId, spatialLayer } = data
   const consumer = network.consumers.find((c) => c.id === consumerId)!
+  if (!consumer)
+    return spark.write({
+      type: MessageTypes.WebRTCConsumerSetLayers.toString(),
+      data: { layersSet: false },
+      id: messageId
+    })
   logger.info('consumer-set-layers: %o, %o', spatialLayer, consumer.appData)
-  await consumer.setPreferredLayers({ spatialLayer })
-  spark.write({ type: MessageTypes.WebRTCConsumerSetLayers.toString(), data: { layersSet: true }, id: messageId })
+  try {
+    await consumer.setPreferredLayers({ spatialLayer })
+    spark.write({ type: MessageTypes.WebRTCConsumerSetLayers.toString(), data: { layersSet: true }, id: messageId })
+  } catch (err) {
+    logger.warn(err)
+    return spark.write({
+      type: MessageTypes.WebRTCConsumerSetLayers.toString(),
+      data: { layersSet: false },
+      id: messageId
+    })
+  }
 }
 
 export async function handleWebRtcResumeProducer(